diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..ebec8ba8c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +**/*~ +*~ diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..ffdbcdf45 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013-2014 David Steele + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md index b25afb5d7..42ebb6468 100644 --- a/README.md +++ b/README.md @@ -2,3 +2,24 @@ pg_backrest =========== Simple Postgres Backup and Restore + +release notes +============= + +v0.01: Backup and archiving are functional + +This version has been put into production at Resonate, so it does work, but there are a number of major caveats. + +* No restore functionality, but the backup directories are consistent Postgres data directories. You'll need to either uncompress the files or turn off compression in the backup. Uncompressed backups on a ZFS (or similar) filesystem are a good option because backups can be restored locally via a snapshot to create logical backups or do spot data recovery. + +* Archiving is single-threaded. This has not posed an issue on our multi-terabyte databases with heavy write volume. Recommend a large WAL volume or to use the async option with a large volume nearby. + +* Backups are multi-threaded, but the Net::OpenSSH library does not appear to be 100% threadsafe so it will very occasionally lock up on a thread. There is an overall process timeout that resolves this issue by killing the process. Yes, very ugly. + +* Checksums are lost on any resumed backup. Only the final backup will record checksum on multiple resumes. Checksums from previous backups are correctly recorded and a full backup will reset everything. + +* The backup.manifest is being written as Storable because Config::IniFile does not seem to handle large files well. Would definitely like to save these as human-readable text. + +* Absolutely no documentation (outside the code). Well, excepting these release notes. + +* Lots of other little things and not so little things. Much refactoring to follow. diff --git a/pg_backrest.conf b/pg_backrest.conf new file mode 100644 index 000000000..ed78c12eb --- /dev/null +++ b/pg_backrest.conf @@ -0,0 +1,38 @@ +[global:command] +#compress=pigz --rsyncable --best --stdout %file% # Ubuntu Linux +compress=/usr/bin/gzip --stdout %file% +decompress=/usr/bin/gzip -dc %file% +#checksum=sha1sum %file% | awk '{print $1}' # Ubuntu Linux +checksum=/usr/bin/shasum %file% | awk '{print $1}' +manifest=/opt/local/bin/gfind %path% -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n' +psql=/Library/PostgreSQL/9.3/bin/psql -X %option% + +[global:log] +level-file=debug +level-console=info + +[global:backup] +user=backrest +host=localhost +path=/Users/backrest/test +archive-required=y +thread-max=2 +thread-timeout=900 + +[global:archive] +path=/Users/dsteele/test +compress-async=y +archive-max-mb=500 + +[global:retention] +full_retention=2 +differential_retention=2 +archive_retention_type=full +archive_retention=2 + +[db] +psql_options=--cluster=9.3/main +path=/Users/dsteele/test/db/common + +[db:command:option] +psql=--port=6001 \ No newline at end of file diff --git a/pg_backrest.pl b/pg_backrest.pl new file mode 100755 index 000000000..29294b24a --- /dev/null +++ b/pg_backrest.pl @@ -0,0 +1,514 @@ +#!/usr/bin/perl + +use threads; + +use strict; +use warnings; + +use File::Basename; +use Getopt::Long; +use Config::IniFiles; +use Carp; + +use lib dirname($0); +use pg_backrest_utility; +use pg_backrest_file; +use pg_backrest_backup; +use pg_backrest_db; + +# Operation constants +use constant +{ + OP_ARCHIVE_PUSH => "archive-push", + OP_ARCHIVE_PULL => "archive-pull", + OP_BACKUP => "backup", + OP_EXPIRE => "expire", +}; + +use constant +{ + CONFIG_SECTION_COMMAND => "command", + CONFIG_SECTION_COMMAND_OPTION => "command:option", + CONFIG_SECTION_LOG => "log", + CONFIG_SECTION_BACKUP => "backup", + CONFIG_SECTION_ARCHIVE => "archive", + CONFIG_SECTION_RETENTION => "retention", + CONFIG_SECTION_STANZA => "stanza", + + CONFIG_KEY_USER => "user", + CONFIG_KEY_HOST => "host", + CONFIG_KEY_PATH => "path", + + CONFIG_KEY_THREAD_MAX => "thread-max", + CONFIG_KEY_THREAD_TIMEOUT => "thread-timeout", + CONFIG_KEY_HARDLINK => "hardlink", + CONFIG_KEY_ARCHIVE_REQUIRED => "archive-required", + CONFIG_KEY_ARCHIVE_MAX_MB => "archive-max-mb", + + CONFIG_KEY_LEVEL_FILE => "level-file", + CONFIG_KEY_LEVEL_CONSOLE => "level-console", + + CONFIG_KEY_COMPRESS => "compress", + CONFIG_KEY_COMPRESS_ASYNC => "compress-async", + CONFIG_KEY_DECOMPRESS => "decompress", + CONFIG_KEY_CHECKSUM => "checksum", + CONFIG_KEY_MANIFEST => "manifest", + CONFIG_KEY_PSQL => "psql" +}; + +# Command line parameters +my $strConfigFile; # Configuration file +my $strStanza; # Stanza in the configuration file to load +my $strType; # Type of backup: full, differential (diff), incremental (incr) + +GetOptions ("config=s" => \$strConfigFile, + "stanza=s" => \$strStanza, + "type=s" => \$strType) + or die("Error in command line arguments\n"); + +# Global variables +my %oConfig; + +#################################################################################################################################### +# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) +#################################################################################################################################### +sub config_load +{ + my $strSection = shift; + my $strKey = shift; + my $bRequired = shift; + my $strDefault = shift; + + # Default is that the key is not required + if (!defined($bRequired)) + { + $bRequired = false; + } + + my $strValue; + + # Look in the default stanza section + if ($strSection eq CONFIG_SECTION_STANZA) + { + $strValue = $oConfig{"${strStanza}"}{"${strKey}"}; + } + # Else look in the supplied section + else + { + # First check the stanza section + $strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"}; + + # If the stanza section value is undefined then check global + if (!defined($strValue)) + { + $strValue = $oConfig{"global:${strSection}"}{"${strKey}"}; + } + } + + if (!defined($strValue) && $bRequired) + { + if (defined($strDefault)) + { + return $strDefault; + } + + confess &log(ERROR, "config value " . (defined($strSection) ? $strSection : "[stanza]") . "->${strKey} is undefined"); + } + + if ($strSection eq CONFIG_SECTION_COMMAND) + { + my $strOption = config_load(CONFIG_SECTION_COMMAND_OPTION, $strKey); + + if (defined($strOption)) + { + $strValue =~ s/\%option\%/${strOption}/g; + } + } + + return $strValue; +} + +#################################################################################################################################### +# SAFE_EXIT - terminate all SSH sessions when the script is terminated +#################################################################################################################################### +sub safe_exit +{ + my $iTotal = backup_thread_kill(); + + confess &log(ERROR, "process was terminated on signal, ${iTotal} threads stopped"); +} + +$SIG{TERM} = \&safe_exit; +$SIG{HUP} = \&safe_exit; +$SIG{INT} = \&safe_exit; + +#################################################################################################################################### +# START MAIN +#################################################################################################################################### +# Get the operation +my $strOperation = $ARGV[0]; + +# Validate the operation +if (!defined($strOperation)) +{ + confess &log(ERROR, "operation is not defined"); +} + +if ($strOperation ne OP_ARCHIVE_PUSH && + $strOperation ne OP_ARCHIVE_PULL && + $strOperation ne OP_BACKUP && + $strOperation ne OP_EXPIRE) +{ + confess &log(ERROR, "invalid operation ${strOperation}"); +} + +# Type should only be specified for backups +if (defined($strType) && $strOperation ne OP_BACKUP) +{ + confess &log(ERROR, "type can only be specified for the backup operation") +} + +#################################################################################################################################### +# LOAD CONFIG FILE +#################################################################################################################################### +if (!defined($strConfigFile)) +{ + $strConfigFile = "/etc/pg_backrest.conf"; +} + +tie %oConfig, 'Config::IniFiles', (-file => $strConfigFile) or confess &log(ERROR, "unable to find config file ${strConfigFile}"); + +# Load and check the cluster +if (!defined($strStanza)) +{ + confess "a backup stanza must be specified - show usage"; +} + +# Set the log levels +log_level_set(uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, "INFO")), + uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, "ERROR"))); + +#################################################################################################################################### +# ARCHIVE-PUSH Command +#################################################################################################################################### +if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL) +{ + # If an archive section has been defined, use that instead of the backup section when operation is OP_ARCHIVE_PUSH + my $strSection = defined(config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH)) ? CONFIG_SECTION_ARCHIVE : CONFIG_SECTION_BACKUP; + + # Get the async compress flag. If compress_async=y then compression is off for the initial push + my $bCompressAsync = config_load($strSection, CONFIG_KEY_COMPRESS_ASYNC, true, "n") eq "n" ? false : true; + + # Get the async compress flag. If compress_async=y then compression is off for the initial push + my $strStopFile; + my $strArchivePath; + + # If logging locally then create the stop archiving file name + if ($strSection eq CONFIG_SECTION_ARCHIVE) + { + $strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH); + $strStopFile = "${strArchivePath}/lock/${strStanza}-archive.stop"; + } + + # Perform the archive-push + if ($strOperation eq OP_ARCHIVE_PUSH) + { + # Call the archive_push function + if (!defined($ARGV[1])) + { + confess &log(ERROR, "source archive file not provided - show usage"); + } + + # If the stop file exists then discard the archive log + if (defined($strStopFile)) + { + if (-e $strStopFile) + { + &log(ERROR, "archive stop file exists ($strStopFile), discarding " . basename($ARGV[1])); + exit 0; + } + } + + # Make sure that archive-push is running locally + if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST))) + { + confess &log(ERROR, "stanza host cannot be set on archive-push - must be run locally on db server"); + } + + # Get the compress flag + my $bCompress = $bCompressAsync ? false : config_load($strSection, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false; + + # Get the checksum flag + my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false; + + # Run file_init_archive - this is the minimal config needed to run archiving + my $oFile = pg_backrest_file->new + ( + strStanza => $strStanza, + bNoCompression => !$bCompress, + strBackupUser => config_load($strSection, CONFIG_KEY_USER), + strBackupHost => config_load($strSection, CONFIG_KEY_HOST), + strBackupPath => config_load($strSection, CONFIG_KEY_PATH, true), + strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum), + strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress), + strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress) + ); + + backup_init + ( + undef, + $oFile, + undef, + undef, + !$bChecksum + ); + + &log(INFO, "pushing archive log " . $ARGV[1] . ($bCompressAsync ? " asynchronously" : "")); + + archive_push($ARGV[1]); + + # Only continue if we are archiving local and a backup server is defined + if (!($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)))) + { + exit 0; + } + + # Set the operation so that archive-pull will be called next + $strOperation = OP_ARCHIVE_PULL; + + # fork and exit the parent process + if (fork()) + { + exit 0; + } + } + + # Perform the archive-pull + if ($strOperation eq OP_ARCHIVE_PULL) + { + # Make sure that archive-pull is running on the db server + if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST))) + { + confess &log(ERROR, "stanza host cannot be set on archive-pull - must be run locally on db server"); + } + + # Create a lock file to make sure archive-pull does not run more than once + my $strLockFile = "${strArchivePath}/lock/${strStanza}-archive.lock"; + + if (!lock_file_create($strLockFile)) + { + &log(DEBUG, "archive-pull process is already running - exiting"); + exit 0 + } + + # Build the basic command string that will be used to modify the command during processing + my $strCommand = $^X . " " . $0 . " --stanza=${strStanza}"; + + # Get the new operational flags + my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false; + my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false; + my $iArchiveMaxMB = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_ARCHIVE_MAX_MB); + + eval + { + # Run file_init_archive - this is the minimal config needed to run archive pulling + my $oFile = pg_backrest_file->new + ( + strStanza => $strStanza, + bNoCompression => !$bCompress, + strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER), + strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST), + strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true), + strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum), + strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress), + strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress), + strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST) + ); + + backup_init + ( + undef, + $oFile, + undef, + undef, + !$bChecksum, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX), + undef, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT) + ); + + # Call the archive_pull function Continue to loop as long as there are files to process. + while (archive_pull($strArchivePath . "/archive/${strStanza}", $strStopFile, $strCommand, $iArchiveMaxMB)) + { + } + }; + + # If there were errors above then start compressing + if ($@) + { + if ($bCompressAsync) + { + &log(ERROR, "error during transfer: $@"); + &log(WARN, "errors during transfer, starting compression"); + + # Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file + my $oFile = pg_backrest_file->new + ( + strStanza => $strStanza, + bNoCompression => false, + strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true), + strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum), + strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress), + strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress), + strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST) + ); + + backup_init + ( + undef, + $oFile, + undef, + undef, + !$bChecksum, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX), + undef, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT) + ); + + archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand, 256); + } + else + { + confess $@; + } + } + + lock_file_remove(); + } + + exit 0; +} + +#################################################################################################################################### +# OPEN THE LOG FILE +#################################################################################################################################### +if (defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST))) +{ + confess &log(ASSERT, "backup/expire operations must be performed locally on the backup server"); +} + +log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/${strStanza}"); + +#################################################################################################################################### +# GET MORE CONFIG INFO +#################################################################################################################################### +# Set the backup type +if (!defined($strType)) +{ + $strType = "incremental"; +} +elsif ($strType eq "diff") +{ + $strType = "differential"; +} +elsif ($strType eq "incr") +{ + $strType = "incremental"; +} +elsif ($strType ne "full" && $strType ne "differential" && $strType ne "incremental") +{ + confess &log(ERROR, "backup type must be full, differential (diff), incremental (incr)"); +} + +# Get the operational flags +my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false; +my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false; + +# Run file_init_archive - the rest of the file config required for backup and restore +my $oFile = pg_backrest_file->new +( + strStanza => $strStanza, + bNoCompression => !$bCompress, + strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER), + strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST), + strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true), + strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER), + strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST), + strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum), + strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress), + strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress), + strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST), + strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL) +); + +my $oDb = pg_backrest_db->new +( + strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER), + strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST), + strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL), + oDbSSH => $oFile->{oDbSSH} +); + +# Run backup_init - parameters required for backup and restore operations +backup_init +( + $oDb, + $oFile, + $strType, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HARDLINK, true, "n") eq "y" ? true : false, + !$bChecksum, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX), + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_ARCHIVE_REQUIRED, true, "y") eq "y" ? true : false, + config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT) +); + +#################################################################################################################################### +# BACKUP +#################################################################################################################################### +if ($strOperation eq OP_BACKUP) +{ + my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-backup.lock"); + + if (!lock_file_create($strLockFile)) + { + &log(DEBUG, "backup process is already running for stanza ${strStanza} - exiting"); + exit 0 + } + + backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH)); + + $strOperation = OP_EXPIRE; + + sleep(30); + + lock_file_remove(); +} + +#################################################################################################################################### +# EXPIRE +#################################################################################################################################### +if ($strOperation eq OP_EXPIRE) +{ + my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-expire.lock"); + + if (!lock_file_create($strLockFile)) + { + &log(DEBUG, "expire process is already running for stanza ${strStanza} - exiting"); + exit 0 + } + + backup_expire + ( + $oFile->path_get(PATH_BACKUP_CLUSTER), + config_load(CONFIG_SECTION_RETENTION, "full_retention"), + config_load(CONFIG_SECTION_RETENTION, "differential_retention"), + config_load(CONFIG_SECTION_RETENTION, "archive_retention_type"), + config_load(CONFIG_SECTION_RETENTION, "archive_retention") + ); + + lock_file_remove(); + + exit 0; +} + +confess &log(ASSERT, "invalid operation ${strOperation} - missing handler block"); \ No newline at end of file diff --git a/pg_backrest_backup.pm b/pg_backrest_backup.pm new file mode 100644 index 000000000..12dc92b30 --- /dev/null +++ b/pg_backrest_backup.pm @@ -0,0 +1,1591 @@ +#################################################################################################################################### +# BACKUP MODULE +#################################################################################################################################### +package pg_backrest_backup; + +use threads; + +use strict; +use warnings; +use Carp; +use File::Basename; +use File::Path qw(remove_tree); +use JSON; +use Scalar::Util qw(looks_like_number); +use Storable; +use Thread::Queue; + +use lib dirname($0); +use pg_backrest_utility; +use pg_backrest_file; +use pg_backrest_db; + +use Exporter qw(import); + +our @EXPORT = qw(backup_init backup_thread_kill archive_push archive_pull archive_compress backup backup_expire archive_list_get); + +my $oDb; +my $oFile; +my $strType = "incremental"; # Type of backup: full, differential (diff), incremental (incr) +my $bHardLink; +my $bNoChecksum; +my $iThreadMax; +my $iThreadLocalMax; +my $iThreadThreshold = 10; +my $iSmallFileThreshold = 65536; +my $bArchiveRequired; +my $iThreadTimeout; + +# Thread variables +my @oThread; +my @oThreadQueue; +my @oMasterQueue; +my %oFileCopyMap; + +#################################################################################################################################### +# BACKUP_INIT +#################################################################################################################################### +sub backup_init +{ + my $oDbParam = shift; + my $oFileParam = shift; + my $strTypeParam = shift; + my $bHardLinkParam = shift; + my $bNoChecksumParam = shift; + my $iThreadMaxParam = shift; + my $bArchiveRequiredParam = shift; + my $iThreadTimeoutParam = shift; + + $oDb = $oDbParam; + $oFile = $oFileParam; + $strType = $strTypeParam; + $bHardLink = $bHardLinkParam; + $bNoChecksum = $bNoChecksumParam; + $iThreadMax = $iThreadMaxParam; + $bArchiveRequired = $bArchiveRequiredParam; + $iThreadTimeout = $iThreadTimeoutParam; + + if (!defined($iThreadMax)) + { + $iThreadMax = 1; + } + + if ($iThreadMax < 1 || $iThreadMax > 32) + { + confess &log(ERROR, "thread_max must be between 1 and 32"); + } +} + +#################################################################################################################################### +# THREAD_INIT +#################################################################################################################################### +sub thread_init +{ + my $iThreadRequestTotal = shift; # Number of threads that were requested + + my $iThreadActualTotal; # Number of actual threads assigned + + if (!defined($iThreadRequestTotal)) + { + $iThreadActualTotal = $iThreadMax; + } + else + { + $iThreadActualTotal = $iThreadRequestTotal < $iThreadMax ? $iThreadRequestTotal : $iThreadMax; + + if ($iThreadActualTotal < 1) + { + $iThreadActualTotal = 1; + } + } + + for (my $iThreadIdx = 0; $iThreadIdx < $iThreadActualTotal; $iThreadIdx++) + { + $oThreadQueue[$iThreadIdx] = Thread::Queue->new(); + $oMasterQueue[$iThreadIdx] = Thread::Queue->new(); + } + + return $iThreadActualTotal; +} + +#################################################################################################################################### +# BACKUP_THREAD_KILL +#################################################################################################################################### +sub backup_thread_kill +{ + my $iTotal = 0; + + for (my $iThreadIdx = 0; $iThreadIdx < scalar @oThread; $iThreadIdx++) + { + if (defined($oThread[$iThreadIdx])) + { + if ($oThread[$iThreadIdx]->is_running()) + { + $oThread[$iThreadIdx]->kill('KILL')->join(); + } + elsif ($oThread[$iThreadIdx]->is_joinable()) + { + $oThread[$iThreadIdx]->join(); + } + + undef($oThread[$iThreadIdx]); + $iTotal++; + } + } + + return($iTotal); +} + +#################################################################################################################################### +# BACKUP_THREAD_COMPLETE +#################################################################################################################################### +sub backup_thread_complete +{ + my $iTimeout = shift; + my $bConfessOnError = shift; + + if (!defined($bConfessOnError)) + { + $bConfessOnError = true; + } + + if (!defined($iTimeout)) + { + &log(WARN, "no thread timeout was set"); + } + + # Wait for all threads to complete and handle errors + my $iThreadComplete = 0; + my $lTimeBegin = time(); + + # Rejoin the threads + while ($iThreadComplete < $iThreadLocalMax) + { + sleep(1); + + # If a timeout has been defined, make sure we have not been running longer than that + if (defined($iTimeout)) + { + if (time() - $lTimeBegin >= $iTimeout) + { + confess &log(ERROR, "threads have been running more than ${iTimeout} seconds, exiting..."); + + #backup_thread_kill(); + + #confess &log(WARN, "all threads have exited, aborting..."); + } + } + + for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + { + if (defined($oThread[$iThreadIdx])) + { + if (defined($oThread[$iThreadIdx]->error())) + { + backup_thread_kill(); + + if ($bConfessOnError) + { + confess &log(ERROR, "error in thread ${iThreadIdx}: check log for details"); + } + else + { + return false; + } + } + + if ($oThread[$iThreadIdx]->is_joinable()) + { + &log(DEBUG, "thread ${iThreadIdx} exited"); + $oThread[$iThreadIdx]->join(); + &log(TRACE, "thread ${iThreadIdx} object undef"); + undef($oThread[$iThreadIdx]); + $iThreadComplete++; + } + } + } + } + + &log(DEBUG, "all threads exited"); + + return true; +} + +#################################################################################################################################### +# ARCHIVE_PUSH +#################################################################################################################################### +sub archive_push +{ + my $strSourceFile = shift; + + # Get the destination file + my $strDestinationFile = basename($strSourceFile); + + # Determine if this is an archive file (don't want to do compression or checksum on .backup files) + my $bArchiveFile = basename($strSourceFile) =~ /^[0-F]{24}$/ ? true : false; + + # Append the checksum (if requested) + if ($bArchiveFile && !$bNoChecksum) + { + $strDestinationFile .= "-" . $oFile->file_hash_get(PATH_DB_ABSOLUTE, $strSourceFile); + } + + # Copy the archive file + $oFile->file_copy(PATH_DB_ABSOLUTE, $strSourceFile, PATH_BACKUP_ARCHIVE, $strDestinationFile, + $bArchiveFile ? undef : true); +} + +#################################################################################################################################### +# ARCHIVE_PULL +#################################################################################################################################### +sub archive_pull +{ + my $strArchivePath = shift; + my $strStopFile = shift; + my $strCommand = shift; + my $iArchiveMaxMB = shift; + + # Load the archive manifest - all the files that need to be pushed + my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strArchivePath); + + # Get all the files to be transferred and calculate the total size + my @stryFile; + my $lFileSize = 0; + my $lFileTotal = 0; + + foreach my $strFile (sort(keys $oManifestHash{name})) + { + if ($strFile =~ /^[0-F]{16}\/[0-F]{24}.*/) + { + push @stryFile, $strFile; + + $lFileSize += $oManifestHash{name}{"$strFile"}{size}; + $lFileTotal++; + } + } + + if (defined($iArchiveMaxMB)) + { + if ($iArchiveMaxMB < int($lFileSize / 1024 / 1024)) + { + &log(ERROR, "local archive store has exceeded limit of ${iArchiveMaxMB}MB, archive logs will be discarded"); + + my $hStopFile; + open($hStopFile, ">", $strStopFile) or confess &log(ERROR, "unable to create stop file file ${strStopFile}"); + close($hStopFile); + } + } + + if ($lFileTotal == 0) + { + &log(DEBUG, "no archive logs to be copied to backup"); + + return 0; + } + + $0 = "${strCommand} archive-push-async " . substr($stryFile[0], 17, 24) . "-" . substr($stryFile[scalar @stryFile - 1], 17, 24); + + # Output files to be moved to backup + &log(INFO, "archive to be copied to backup total ${lFileTotal}, size " . file_size_format($lFileSize)); + + # # Init the thread variables + # $iThreadLocalMax = thread_init(int($lFileTotal / $iThreadThreshold) + 1); + # my $iThreadIdx = 0; + # + # &log(DEBUG, "actual threads ${iThreadLocalMax}/${iThreadMax}"); + # + # # Distribute files among the threads + # foreach my $strFile (sort @stryFile) + # { + # $oThreadQueue[$iThreadIdx]->enqueue($strFile); + # + # $iThreadIdx = ($iThreadIdx + 1 == $iThreadLocalMax) ? 0 : $iThreadIdx + 1; + # } + # + # # End each thread queue and start the thread + # for ($iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + # { + # $oThreadQueue[$iThreadIdx]->enqueue(undef); + # $oThread[$iThreadIdx] = threads->create(\&archive_pull_copy_thread, $iThreadIdx, $strArchivePath); + # } + # + # backup_thread_complete($iThreadTimeout); + + # Transfer each file + foreach my $strFile (sort @stryFile) + { + &log(INFO, "backing up archive file ${strFile}"); + + my $strArchiveFile = "${strArchivePath}/${strFile}"; + + # Copy the file + $oFile->file_copy(PATH_DB_ABSOLUTE, $strArchiveFile, + PATH_BACKUP_ARCHIVE, basename($strFile)); + + # Remove the source archive file + unlink($strArchiveFile) or confess &log(ERROR, "unable to remove ${strArchiveFile}"); + } + + # Find the archive paths that need to be removed + my $strPathMax = substr((sort {$b cmp $a} @stryFile)[0], 0, 16); + + &log(DEBUG, "local archive path max = ${strPathMax}"); + + foreach my $strPath ($oFile->file_list_get(PATH_DB_ABSOLUTE, $strArchivePath, "^[0-F]{16}\$")) + { + if ($strPath lt $strPathMax) + { + &log(DEBUG, "removing local archive path ${strPath}"); + rmdir($strArchivePath . "/" . $strPath) or confess &log(ERROR, "unable to remove archive path ${strPath}, is it empty?"); + } + } + + # Return number of files indicating that processing should continue + return $lFileTotal; +} + +# sub archive_pull_copy_thread +# { +# my @args = @_; +# +# my $iThreadIdx = $args[0]; +# my $strArchivePath = $args[1]; +# +# my $oFileThread = $oFile->clone($iThreadIdx); # Thread local file object +# +# # When a KILL signal is received, immediately abort +# $SIG{'KILL'} = sub {threads->exit();}; +# +# while (my $strFile = $oThreadQueue[$iThreadIdx]->dequeue()) +# { +# &log(INFO, "thread ${iThreadIdx} backing up archive file ${strFile}"); +# +# my $strArchiveFile = "${strArchivePath}/${strFile}"; +# +# # Copy the file +# $oFileThread->file_copy(PATH_DB_ABSOLUTE, $strArchiveFile, +# PATH_BACKUP_ARCHIVE, basename($strFile), +# undef, undef, +# undef); # cannot set permissions remotely yet $oFile->{strDefaultFilePermission}); +# +# # Remove the source archive file +# unlink($strArchiveFile) or confess &log(ERROR, "unable to remove ${strArchiveFile}"); +# } +# } + +sub archive_compress +{ + my $strArchivePath = shift; + my $strCommand = shift; + my $iFileCompressMax = shift; + + # Load the archive manifest - all the files that need to be pushed + my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strArchivePath); + + # Get all the files to be compressed and calculate the total size + my @stryFile; + my $lFileSize = 0; + my $lFileTotal = 0; + + foreach my $strFile (sort(keys $oManifestHash{name})) + { + if ($strFile =~ /^[0-F]{16}\/[0-F]{24}(\-[0-f]+){0,1}$/) + { + push @stryFile, $strFile; + + $lFileSize += $oManifestHash{name}{"$strFile"}{size}; + $lFileTotal++; + + if ($lFileTotal >= $iFileCompressMax) + { + last; + } + } + } + + if ($lFileTotal == 0) + { + &log(DEBUG, "no archive logs to be compressed"); + + return; + } + + $0 = "${strCommand} archive-compress-async " . substr($stryFile[0], 17, 24) . "-" . substr($stryFile[scalar @stryFile - 1], 17, 24); + + # Output files to be compressed + &log(INFO, "archive to be compressed total ${lFileTotal}, size " . file_size_format($lFileSize)); + + # Init the thread variables + $iThreadLocalMax = thread_init(int($lFileTotal / $iThreadThreshold) + 1); + my $iThreadIdx = 0; + + # Distribute files among the threads + foreach my $strFile (sort @stryFile) + { + $oThreadQueue[$iThreadIdx]->enqueue($strFile); + + $iThreadIdx = ($iThreadIdx + 1 == $iThreadLocalMax) ? 0 : $iThreadIdx + 1; + } + + # End each thread queue and start the thread + for ($iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + { + $oThreadQueue[$iThreadIdx]->enqueue(undef); + $oThread[$iThreadIdx] = threads->create(\&archive_pull_compress_thread, $iThreadIdx, $strArchivePath); + } + + # Complete the threads + backup_thread_complete($iThreadTimeout); +} + +sub archive_pull_compress_thread +{ + my @args = @_; + + my $iThreadIdx = $args[0]; + my $strArchivePath = $args[1]; + + my $oFileThread = $oFile->clone($iThreadIdx); # Thread local file object + + # When a KILL signal is received, immediately abort + $SIG{'KILL'} = sub {threads->exit();}; + + while (my $strFile = $oThreadQueue[$iThreadIdx]->dequeue()) + { + &log(INFO, "thread ${iThreadIdx} compressing archive file ${strFile}"); + + # Compress the file + $oFileThread->file_compress(PATH_DB_ABSOLUTE, "${strArchivePath}/${strFile}"); + } +} + +#################################################################################################################################### +# BACKUP_REGEXP_GET - Generate a regexp depending on the backups that need to be found +#################################################################################################################################### +sub backup_regexp_get +{ + my $bFull = shift; + my $bDifferential = shift; + my $bIncremental = shift; + + if (!$bFull && !$bDifferential && !$bIncremental) + { + confess &log(ERROR, 'one parameter must be true'); + } + + my $strDateTimeRegExp = "[0-9]{8}\\-[0-9]{6}"; + my $strRegExp = "^"; + + if ($bFull || $bDifferential || $bIncremental) + { + $strRegExp .= $strDateTimeRegExp . "F"; + } + + if ($bDifferential || $bIncremental) + { + if ($bFull) + { + $strRegExp .= "(\\_"; + } + + $strRegExp .= $strDateTimeRegExp; + + if ($bDifferential && $bIncremental) + { + $strRegExp .= "(D|I)"; + } + elsif ($bDifferential) + { + $strRegExp .= "D"; + } + else + { + $strRegExp .= "I"; + } + + if ($bFull) + { + $strRegExp .= "){0,1}"; + } + } + + $strRegExp .= "\$"; + +# &log(DEBUG, "backup_regexp_get($bFull, $bDifferential, $bIncremental): $strRegExp"); + + return $strRegExp; +} + +#################################################################################################################################### +# BACKUP_TYPE_FIND - Find the last backup depending on the type +#################################################################################################################################### +sub backup_type_find +{ + my $strType = shift; + my $strBackupClusterPath = shift; + my $strDirectory; + + if ($strType eq 'incremental') + { + $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"))[0]; + } + + if (!defined($strDirectory) && $strType ne "full") + { + $strDirectory = ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"))[0]; + } + + return $strDirectory; +} + +#################################################################################################################################### +# BACKUP_MANIFEST_LOAD - Load the backup manifest +#################################################################################################################################### +sub backup_manifest_load +{ + my $strBackupManifestFile = shift; + + return %{retrieve($strBackupManifestFile) or confess &log(ERROR, "unable to read ${strBackupManifestFile}")}; +} + +#################################################################################################################################### +# BACKUP_MANIFEST_SAVE - Save the backup manifest +#################################################################################################################################### +sub backup_manifest_save +{ + my $strBackupManifestFile = shift; + my $oBackupManifestRef = shift; + + store($oBackupManifestRef, $strBackupManifestFile) or confess &log(ERROR, "unable to open ${strBackupManifestFile}"); +} + +#################################################################################################################################### +# BACKUP_FILE_NOT_IN_MANIFEST - Find all files in a backup path that are not in the supplied manifest +#################################################################################################################################### +sub backup_file_not_in_manifest +{ + my $strPathType = shift; + my $oManifestRef = shift; + + my %oFileHash = $oFile->manifest_get($strPathType); + my @stryFile; + my $iFileTotal = 0; + + foreach my $strName (sort(keys $oFileHash{name})) + { + # Ignore certain files that will never be in the manifest + if ($strName eq "backup.manifest" || + $strName eq ".") + { + next; + } + + # Get the base directory + my $strBasePath = (split("/", $strName))[0]; + + if ($strBasePath eq $strName) + { + my $strSection = $strBasePath eq "tablespace" ? "base:tablespace" : "${strBasePath}:path"; + + if (defined(${$oManifestRef}{"${strSection}"})) + { + next; + } + } + else + { + my $strPath = substr($strName, length($strBasePath) + 1); + + # Create the section from the base path + my $strSection = $strBasePath; + + if ($strSection eq "tablespace") + { + my $strTablespace = (split("/", $strPath))[0]; + + $strSection = $strSection . ":" . $strTablespace; + + if ($strTablespace eq $strPath) + { + if (defined(${$oManifestRef}{"${strSection}:path"})) + { + next; + } + } + + $strPath = substr($strPath, length($strTablespace) + 1); + } + + my $cType = $oFileHash{name}{"${strName}"}{type}; + + if ($cType eq "d") + { + if (defined(${$oManifestRef}{"${strSection}:path"}{"${strPath}"})) + { + next; + } + } + else + { + if (defined(${$oManifestRef}{"${strSection}:file"}{"${strPath}"})) + { + if (${$oManifestRef}{"${strSection}:file"}{"${strPath}"}{size} == + $oFileHash{name}{"${strName}"}{size} && + ${$oManifestRef}{"${strSection}:file"}{"${strPath}"}{modification_time} == + $oFileHash{name}{"${strName}"}{modification_time}) + { + ${$oManifestRef}{"${strSection}:file"}{"${strPath}"}{exists} = true; + next; + } + } + } + } + + $stryFile[$iFileTotal] = $strName; + $iFileTotal++; + } + + return @stryFile; +} + +#################################################################################################################################### +# BACKUP_TMP_CLEAN +# +# Cleans the temp directory from a previous failed backup so it can be reused +#################################################################################################################################### +sub backup_tmp_clean +{ + my $oManifestRef = shift; + + &log(INFO, "cleaning backup tmp path"); + + # Remove the pg_xlog directory since it contains nothing useful for the new backup + if (-e $oFile->path_get(PATH_BACKUP_TMP, "base/pg_xlog")) + { + remove_tree($oFile->path_get(PATH_BACKUP_TMP, "base/pg_xlog")) or confess &log(ERROR, "unable to delete tmp pg_xlog path"); + } + + # Remove the pg_tblspc directory since it is trivial to rebuild, but hard to compare + if (-e $oFile->path_get(PATH_BACKUP_TMP, "base/pg_tblspc")) + { + remove_tree($oFile->path_get(PATH_BACKUP_TMP, "base/pg_tblspc")) or confess &log(ERROR, "unable to delete tmp pg_tblspc path"); + } + + # Get the list of files that should be deleted from temp + my @stryFile = backup_file_not_in_manifest(PATH_BACKUP_TMP, $oManifestRef); + + foreach my $strFile (sort {$b cmp $a} @stryFile) + { + my $strDelete = $oFile->path_get(PATH_BACKUP_TMP, $strFile); + + # If a path then delete it, all the files should have already been deleted since we are going in reverse order + if (-d $strDelete) + { + &log(DEBUG, "remove path ${strDelete}"); + rmdir($strDelete) or confess &log(ERROR, "unable to delete path ${strDelete}, is it empty?"); + } + # Else delete a file + else + { + &log(DEBUG, "remove file ${strDelete}"); + unlink($strDelete) or confess &log(ERROR, "unable to delete file ${strDelete}"); + } + } +} + +#################################################################################################################################### +# BACKUP_MANIFEST_BUILD - Create the backup manifest +#################################################################################################################################### +sub backup_manifest_build +{ + my $strCommandManifest = shift; + my $strDbClusterPath = shift; + my $oBackupManifestRef = shift; + my $oLastManifestRef = shift; + my $oTablespaceMapRef = shift; + my $strLevel = shift; + + if (!defined($strLevel)) + { + $strLevel = "base"; + } + + my %oManifestHash = $oFile->manifest_get(PATH_DB_ABSOLUTE, $strDbClusterPath); + my $strName; + + foreach $strName (sort(keys $oManifestHash{name})) + { + # Skip certain files during backup + if ($strName =~ /^pg\_xlog\/.*/ || # pg_xlog/ - this will be reconstructed + $strName =~ /^postmaster\.pid$/) # postmaster.pid - to avoid confusing postgres when restoring + { + next; + } + + my $cType = $oManifestHash{name}{"${strName}"}{type}; + my $strLinkDestination = $oManifestHash{name}{"${strName}"}{link_destination}; + my $strSection = "${strLevel}:path"; + + if ($cType eq "f") + { + $strSection = "${strLevel}:file"; + } + elsif ($cType eq "l") + { + $strSection = "${strLevel}:link"; + } + elsif ($cType ne "d") + { + confess &log(ASSERT, "unrecognized file type $cType for file $strName"); + } + + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{user} = $oManifestHash{name}{"${strName}"}{user}; + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{group} = $oManifestHash{name}{"${strName}"}{group}; + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{permission} = $oManifestHash{name}{"${strName}"}{permission}; + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} = + (split("\\.", $oManifestHash{name}{"${strName}"}{modification_time}))[0]; + + if ($cType eq "f") + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{size} = $oManifestHash{name}{"${strName}"}{size}; + } + + if ($cType eq "f" || $cType eq "l") + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{inode} = $oManifestHash{name}{"${strName}"}{inode}; + + if (defined(${$oLastManifestRef}{"${strSection}"}{"$strName"})) + { + my $bSizeMatch = true; + + if ($cType eq "f") + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{size} = $oManifestHash{name}{"${strName}"}{size}; + + $bSizeMatch = ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{size} == + ${$oLastManifestRef}{"${strSection}"}{"$strName"}{size}; + } + + if ($bSizeMatch && + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{modification_time} == + ${$oLastManifestRef}{"${strSection}"}{"$strName"}{modification_time}) + { + if (defined(${$oLastManifestRef}{"${strSection}"}{"$strName"}{reference})) + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{reference} = + ${$oLastManifestRef}{"${strSection}"}{"$strName"}{reference}; + } + else + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{reference} = + ${$oLastManifestRef}{backup}{label}; + } + + my $strReference = ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{reference}; + + if (!defined(${$oBackupManifestRef}{backup}{reference})) + { + ${$oBackupManifestRef}{backup}{reference} = $strReference; + } + else + { + if (${$oBackupManifestRef}{backup}{reference} !~ /$strReference/) + { + ${$oBackupManifestRef}{backup}{reference} .= ",$strReference"; + } + } + } + } + } + + if ($cType eq "l") + { + ${$oBackupManifestRef}{"${strSection}"}{"$strName"}{link_destination} = + $oManifestHash{name}{"${strName}"}{link_destination}; + + if (index($strName, 'pg_tblspc/') == 0 && $strLevel eq "base") + { + my $strTablespaceOid = basename($strName); + my $strTablespaceName = ${$oTablespaceMapRef}{oid}{"$strTablespaceOid"}{name}; + + ${$oBackupManifestRef}{"${strLevel}:tablespace"}{"${strTablespaceName}"}{oid} = $strTablespaceOid; + ${$oBackupManifestRef}{"${strLevel}:tablespace"}{"${strTablespaceName}"}{path} = $strLinkDestination; + + backup_manifest_build($strCommandManifest, $strLinkDestination, $oBackupManifestRef, $oLastManifestRef, + $oTablespaceMapRef, "tablespace:${strTablespaceName}"); + } + } + } +} + +#################################################################################################################################### +# BACKUP_FILE - Performs the file level backup +# +# Uses the information in the manifest to determine which files need to be copied. Directories and tablespace links are only +# created when needed, except in the case of a full backup or if hardlinks are requested. +#################################################################################################################################### +sub backup_file +{ + my $strBackupPath = shift; # Path where the final backup will go (e.g. 20120101F) + my $strDbClusterPath = shift; # Database base data path + my $oBackupManifestRef = shift; # Manifest for the current backup + + # Variables used for parallel copy + my $lTablespaceIdx = 0; + my $lFileTotal = 0; + my $lFileLargeSize = 0; + my $lFileLargeTotal = 0; + my $lFileSmallSize = 0; + my $lFileSmallTotal = 0; + + # Decide if all the paths will be created in advance + my $bPathCreate = $bHardLink || $strType eq "full"; + + # Iterate through the path sections of the manifest to backup + my $strSectionPath; + + foreach $strSectionPath (sort(keys $oBackupManifestRef)) + { + # Skip non-path sections + if ($strSectionPath !~ /\:path$/) + { + next; + } + + # Determine the source and destination backup paths + my $strBackupSourcePath; # Absolute path to the database base directory or tablespace to backup + my $strBackupDestinationPath; # Relative path to the backup directory where the data will be stored + my $strSectionFile; # Manifest section that contains the file data + + # Process the base database directory + if ($strSectionPath =~ /^base\:/) + { + $lTablespaceIdx++; + $strBackupSourcePath = $strDbClusterPath; + $strBackupDestinationPath = "base"; + $strSectionFile = "base:file"; + + # Create the archive log directory + $oFile->path_create(PATH_BACKUP_TMP, "base/pg_xlog"); + } + # Process each tablespace + elsif ($strSectionPath =~ /^tablespace\:/) + { + $lTablespaceIdx++; + my $strTablespaceName = (split(":", $strSectionPath))[1]; + $strBackupSourcePath = ${$oBackupManifestRef}{"base:tablespace"}{"${strTablespaceName}"}{path}; + $strBackupDestinationPath = "tablespace/${strTablespaceName}"; + $strSectionFile = "tablespace:${strTablespaceName}:file"; + + # Create the tablespace directory and link + if ($bPathCreate) + { + $oFile->path_create(PATH_BACKUP_TMP, $strBackupDestinationPath); + + $oFile->link_create(PATH_BACKUP_TMP, ${strBackupDestinationPath}, + PATH_BACKUP_TMP, + "base/pg_tblspc/" . ${$oBackupManifestRef}{"base:tablespace"}{"${strTablespaceName}"}{oid}, + false, true); + } + } + else + { + confess &log(ASSERT, "cannot find type for section ${strSectionPath}"); + } + + # Create all the sub paths if this is a full backup or hardlinks are requested + if ($bPathCreate) + { + my $strPath; + + foreach $strPath (sort(keys ${$oBackupManifestRef}{"${strSectionPath}"})) + { + if (defined(${$oBackupManifestRef}{"${strSectionPath}"}{"$strPath"}{exists})) + { + &log(TRACE, "path ${strPath} already exists from previous backup attempt"); + ${$oBackupManifestRef}{"${strSectionPath}"}{"$strPath"}{exists} = undef; + } + else + { + $oFile->path_create(PATH_BACKUP_TMP, "${strBackupDestinationPath}/${strPath}", + ${$oBackupManifestRef}{"${strSectionPath}"}{"$strPath"}{permission}); + } + } + } + + # Possible for the path section to exist with no files (i.e. empty tablespace) + if (!defined(${$oBackupManifestRef}{"${strSectionFile}"})) + { + next; + } + + # Iterate through the files for each backup source path + my $strFile; + + foreach $strFile (sort(keys ${$oBackupManifestRef}{"${strSectionFile}"})) + { + my $strBackupSourceFile = "${strBackupSourcePath}/${strFile}"; + + if (defined(${$oBackupManifestRef}{"${strSectionFile}"}{"$strFile"}{exists})) + { + &log(TRACE, "file ${strFile} already exists from previous backup attempt"); + ${$oBackupManifestRef}{"${strSectionPath}"}{"$strFile"}{exists} = undef; + } + else + { + # If the file has a reference it does not need to be copied since it can be retrieved from the referenced backup. + # However, if hard-linking is turned on the link will need to be created + my $strReference = ${$oBackupManifestRef}{"${strSectionFile}"}{"$strFile"}{reference}; + + if (defined($strReference)) + { + # If hardlinking is turned on then create a hardlink for files that have not changed since the last backup + if ($bHardLink) + { + &log(DEBUG, "hard-linking ${strBackupSourceFile} from ${strReference}"); + + $oFile->link_create(PATH_BACKUP_CLUSTER, "${strReference}/${strBackupDestinationPath}/${strFile}", + PATH_BACKUP_TMP, "${strBackupDestinationPath}/${strFile}", true, false, !$bPathCreate); + } + } + # Else copy/compress the file and generate a checksum + else + { + my $lFileSize = ${$oBackupManifestRef}{"${strSectionFile}"}{"$strFile"}{size}; + + # Setup variables needed for threaded copy + $lFileTotal++; + $lFileLargeSize += $lFileSize > $iSmallFileThreshold ? $lFileSize : 0; + $lFileLargeTotal += $lFileSize > $iSmallFileThreshold ? 1 : 0; + $lFileSmallSize += $lFileSize <= $iSmallFileThreshold ? $lFileSize : 0; + $lFileSmallTotal += $lFileSize <= $iSmallFileThreshold ? 1 : 0; + + # Load the hash used by threaded copy + my $strKey = sprintf("ts%012x-fs%012x-fn%012x", $lTablespaceIdx, + $lFileSize, $lFileTotal); + + $oFileCopyMap{"${strKey}"}{db_file} = $strBackupSourceFile; + $oFileCopyMap{"${strKey}"}{file_section} = $strSectionFile; + $oFileCopyMap{"${strKey}"}{file} = ${strFile}; + $oFileCopyMap{"${strKey}"}{backup_file} = "${strBackupDestinationPath}/${strFile}"; + $oFileCopyMap{"${strKey}"}{size} = $lFileSize; + $oFileCopyMap{"${strKey}"}{modification_time} = + ${$oBackupManifestRef}{"${strSectionFile}"}{"$strFile"}{modification_time}; + } + } + } + } + + # Build the thread queues + $iThreadLocalMax = thread_init(int($lFileTotal / $iThreadThreshold) + 1); + &log(DEBUG, "actual threads ${iThreadLocalMax}/${iThreadMax}"); + + # Initialize the thread size array + my @oyThreadData; + + for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + { + $oyThreadData[$iThreadIdx]{size} = 0; + $oyThreadData[$iThreadIdx]{total} = 0; + $oyThreadData[$iThreadIdx]{large_size} = 0; + $oyThreadData[$iThreadIdx]{large_total} = 0; + $oyThreadData[$iThreadIdx]{small_size} = 0; + $oyThreadData[$iThreadIdx]{small_total} = 0; + } + + # Assign files to each thread queue + my $iThreadFileSmallIdx = 0; + my $iThreadFileSmallTotalMax = int($lFileSmallTotal / $iThreadLocalMax); + + my $iThreadFileLargeIdx = 0; + my $fThreadFileLargeSizeMax = $lFileLargeSize / $iThreadLocalMax; + + &log(INFO, "file total ${lFileTotal}"); + &log(INFO, "file small total ${lFileSmallTotal}, small size: " . file_size_format($lFileSmallSize) . + ", small thread avg total " . file_size_format(int($iThreadFileSmallTotalMax))); + &log(INFO, "file large total ${lFileLargeTotal}, large size: " . file_size_format($lFileLargeSize) . + ", large thread avg size " . file_size_format(int($fThreadFileLargeSizeMax))); + + foreach my $strFile (sort (keys %oFileCopyMap)) + { + my $lFileSize = $oFileCopyMap{"${strFile}"}{size}; + + if ($lFileSize > $iSmallFileThreshold) + { + $oThreadQueue[$iThreadFileLargeIdx]->enqueue($strFile); + + $oyThreadData[$iThreadFileLargeIdx]{large_size} += $lFileSize; + $oyThreadData[$iThreadFileLargeIdx]{large_total}++; + $oyThreadData[$iThreadFileLargeIdx]{size} += $lFileSize; + + if ($oyThreadData[$iThreadFileLargeIdx]{large_size} >= $fThreadFileLargeSizeMax && + $iThreadFileLargeIdx < $iThreadLocalMax - 1) + { + $iThreadFileLargeIdx++; + } + } + else + { + $oThreadQueue[$iThreadFileSmallIdx]->enqueue($strFile); + + $oyThreadData[$iThreadFileSmallIdx]{small_size} += $lFileSize; + $oyThreadData[$iThreadFileSmallIdx]{small_total}++; + $oyThreadData[$iThreadFileSmallIdx]{size} += $lFileSize; + + if ($oyThreadData[$iThreadFileSmallIdx]{small_total} >= $iThreadFileSmallTotalMax && + $iThreadFileSmallIdx < $iThreadLocalMax - 1) + { + $iThreadFileSmallIdx++; + } + } + } + + # End each thread queue and start the backu_file threads + for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + { + # Output info about how much work each thread is going to do + &log(INFO, "thread ${iThreadIdx} large total $oyThreadData[$iThreadIdx]{large_total}, " . + "size $oyThreadData[$iThreadIdx]{large_size}"); + &log(INFO, "thread ${iThreadIdx} small total $oyThreadData[$iThreadIdx]{small_total}, " . + "size $oyThreadData[$iThreadIdx]{small_size}"); + + # End each queue + $oThreadQueue[$iThreadIdx]->enqueue(undef); + + # Start the thread + $oThread[$iThreadIdx] = threads->create(\&backup_file_thread, $iThreadIdx, !$bNoChecksum, !$bPathCreate, + $oyThreadData[$iThreadIdx]{size}); + } + + # Wait for the threads to complete + backup_thread_complete($iThreadTimeout); + + # Read the messages that we passed back from the threads. These should be two types: + # 1) remove - files that were skipped because they were removed from the database during backup + # 2) checksum - file checksums calculated by the threads + for (my $iThreadIdx = 0; $iThreadIdx < $iThreadLocalMax; $iThreadIdx++) + { + while (my $strMessage = $oMasterQueue[$iThreadIdx]->dequeue_nb()) + { + &log (DEBUG, "message received in master queue: ${strMessage}"); + + # Split the message. Currently using | as the split character. Not ideal, but it will do for now. + my @strSplit = split(/\|/, $strMessage); + + my $strCommand = $strSplit[0]; # Command to execute on a file + my $strFileSection = $strSplit[1]; # File section where the file is located + my $strFile = $strSplit[2]; # The file to act on + + # These three parts are required + if (!defined($strCommand) || !defined($strFileSection) || !defined($strFile)) + { + confess &log(ASSERT, "thread messages must have strCommand, strFileSection and strFile defined"); + } + + &log (DEBUG, "command = ${strCommand}, file_section = ${strFileSection}, file = ${strFile}"); + + # If command is "remove" then mark the skipped file in the manifest + if ($strCommand eq "remove") + { + delete ${$oBackupManifestRef}{"${strFileSection}"}{"$strFile"}; + + &log (INFO, "marked skipped ${strFileSection}:${strFile} from the manifest"); + } + # If command is "checksum" then record the checksum in the manifest + elsif ($strCommand eq "checksum") + { + my $strChecksum = $strSplit[3]; # File checksum calculated by the thread + + # Checksum must be defined + if (!defined($strChecksum)) + { + confess &log(ASSERT, "thread checksum messages must have strChecksum defined"); + } + + ${$oBackupManifestRef}{"${strFileSection}"}{"$strFile"}{checksum} = $strChecksum; + + # Log the checksum + &log (DEBUG, "write checksum ${strFileSection}:${strFile} into manifest: ${strChecksum}"); + } + } + } +} + +sub backup_file_thread +{ + my @args = @_; + + my $iThreadIdx = $args[0]; # Defines the index of this thread + my $bChecksum = $args[1]; # Should checksums be generated on files after they have been backed up? + my $bPathCreate = $args[2]; # Should paths be created automatically? + my $lSizeTotal = $args[3]; # Total size of the files to be copied by this thread + + my $lSize = 0; # Size of files currently copied by this thread + my $strLog; # Store the log message + my $oFileThread = $oFile->clone($iThreadIdx); # Thread local file object + + # When a KILL signal is received, immediately abort + $SIG{'KILL'} = sub {threads->exit();}; + + # Iterate through all the files in this thread's queue to be copied from the database to the backup + while (my $strFile = $oThreadQueue[$iThreadIdx]->dequeue()) + { + # Add the size of the current file to keep track of percent complete + $lSize += $oFileCopyMap{$strFile}{size}; + + # Output information about the file to be copied + $strLog = "thread ${iThreadIdx} backed up file $oFileCopyMap{$strFile}{db_file} (" . + file_size_format($oFileCopyMap{$strFile}{size}) . + ($lSizeTotal > 0 ? ", " . int($lSize * 100 / $lSizeTotal) . "%" : "") . ")"; + + # Copy the file from the database to the backup + unless($oFileThread->file_copy(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file}, + PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, + undef, $oFileCopyMap{$strFile}{modification_time}, + undef, $bPathCreate, false)) + { + &log(DEBUG, "thread ${iThreadIdx} unable to copy file: " . $oFileCopyMap{$strFile}{db_file}); + + # If the copy fails then see if the file still exists on the database + if (!$oFileThread->file_exists(PATH_DB_ABSOLUTE, $oFileCopyMap{$strFile}{db_file})) + { + # If it is missing then the database must have removed it (or is now corrupt) + &log(INFO, "thread ${iThreadIdx} skipped file removed by database: " . $oFileCopyMap{$strFile}{db_file}); + + # Remove the destination file and the temp file just in case they had already been written + $oFileThread->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}, true); + $oFileThread->file_remove(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); + } + + # Write a message into the master queue to have the file removed from the manifest + $oMasterQueue[$iThreadIdx]->enqueue("remove|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}"); + + # Move on to the next file + next; + } + + # Generate checksum for file if requested + if ($bChecksum && $lSize != 0) + { + # Generate the checksum + my $strChecksum = $oFileThread->file_hash_get(PATH_BACKUP_TMP, $oFileCopyMap{$strFile}{backup_file}); + + # Write the checksum message into the master queue + $oMasterQueue[$iThreadIdx]->enqueue("checksum|$oFileCopyMap{$strFile}{file_section}|$oFileCopyMap{$strFile}{file}|${strChecksum}"); + + &log(INFO, $strLog . " checksum ${strChecksum}"); + } + else + { + &log(INFO, $strLog); + } + } + + &log(DEBUG, "thread ${iThreadIdx} exiting"); +} + +#################################################################################################################################### +# BACKUP +# +# Performs the entire database backup. +#################################################################################################################################### +sub backup +{ + my $strDbClusterPath = shift; + + # Not supporting remote backup hosts yet + if ($oFile->is_remote(PATH_BACKUP)) + { + confess &log(ERROR, "remote backup host not currently supported"); + } + + if (!defined($strDbClusterPath)) + { + confess &log(ERROR, "cluster data path is not defined"); + } + + &log(DEBUG, "cluster path is $strDbClusterPath"); + + # Create the cluster backup path + $oFile->path_create(PATH_BACKUP_CLUSTER); + + # Find the previous backup based on the type + my $strBackupLastPath = backup_type_find($strType, $oFile->path_get(PATH_BACKUP_CLUSTER)); + + my %oLastManifest; + + if (defined($strBackupLastPath)) + { + %oLastManifest = backup_manifest_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strBackupLastPath/backup.manifest"); + + if (!defined($oLastManifest{backup}{label})) + { + confess &log(ERROR, "unable to find label in backup $strBackupLastPath"); + } + + &log(INFO, "last backup label: $oLastManifest{backup}{label}"); + } + + # Create the path for the new backup + my $strBackupPath; + + if ($strType eq "full" || !defined($strBackupLastPath)) + { + $strBackupPath = date_string_get() . "F"; + $strType = "full"; + } + else + { + $strBackupPath = substr($strBackupLastPath, 0, 16); + + $strBackupPath .= "_" . date_string_get(); + + if ($strType eq "differential") + { + $strBackupPath .= "D"; + } + else + { + $strBackupPath .= "I"; + } + } + + &log(INFO, "new backup label: ${strBackupPath}"); + + # Build backup tmp and config + my $strBackupTmpPath = $oFile->path_get(PATH_BACKUP_TMP); + my $strBackupConfFile = $oFile->path_get(PATH_BACKUP_TMP, "backup.manifest"); + + # Start backup + my %oBackupManifest; + ${oBackupManifest}{backup}{label} = $strBackupPath; + + my $strArchiveStart = $oDb->backup_start($strBackupPath); + ${oBackupManifest}{backup}{"archive-start"} = $strArchiveStart; + + &log(INFO, 'archive start: ' . ${oBackupManifest}{backup}{"archive-start"}); + + # Build the backup manifest + my %oTablespaceMap = $oDb->tablespace_map_get(); + + backup_manifest_build($oFile->{strCommandManifest}, $strDbClusterPath, \%oBackupManifest, \%oLastManifest, \%oTablespaceMap); + + # If the backup tmp path already exists, remove invalid files + if (-e $strBackupTmpPath) + { + &log(WARN, "aborted backup already exists, will be cleaned to remove invalid files and resumed"); + + # Clean the old backup tmp path + backup_tmp_clean(\%oBackupManifest); + } + # Else create the backup tmp path + else + { + &log(DEBUG, "creating backup path $strBackupTmpPath"); + $oFile->path_create(PATH_BACKUP_TMP); + } + + # Save the backup conf file first time - so we can see what is happening in the backup + backup_manifest_save($strBackupConfFile, \%oBackupManifest); + + # Perform the backup + backup_file($strBackupPath, $strDbClusterPath, \%oBackupManifest); + + # Stop backup + my $strArchiveStop = $oDb->backup_stop(); + + ${oBackupManifest}{backup}{"archive-stop"} = $strArchiveStop; + &log(INFO, 'archive stop: ' . ${oBackupManifest}{backup}{"archive-stop"}); + + # Need to remove empty directories that were caused by skipped files + # !!! DO IT + + # If archive logs are required to complete the backup, then fetch them. This is the default, but can be overridden if the + # archive logs are going to a different server. Be careful here because there is no way to verify that the backup will be + # consistent - at least not in this routine. + if ($bArchiveRequired) + { + # Save the backup conf file second time - before getting archive logs in case that fails + backup_manifest_save($strBackupConfFile, \%oBackupManifest); + + # After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent + &log(DEBUG, "retrieving archive logs ${strArchiveStart}:${strArchiveStop}"); + my @stryArchive = archive_list_get($strArchiveStart, $strArchiveStop, $oDb->version_get() < 9.3); + + foreach my $strArchive (@stryArchive) + { + my $strArchivePath = dirname($oFile->path_get(PATH_BACKUP_ARCHIVE, $strArchive)); + + wait_for_file($strArchivePath, "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$", 600); + + my @stryArchiveFile = $oFile->file_list_get(PATH_BACKUP_ABSOLUTE, $strArchivePath, + "^${strArchive}(-[0-f]+){0,1}(\\.$oFile->{strCompressExtension}){0,1}\$"); + + if (scalar @stryArchiveFile != 1) + { + confess &log(ERROR, "Zero or more than one file found for glob: ${strArchivePath}"); + } + + &log(DEBUG, "archiving: ${strArchive} (${stryArchiveFile[0]})"); + + $oFile->file_copy(PATH_BACKUP_ARCHIVE, $stryArchiveFile[0], PATH_BACKUP_TMP, "base/pg_xlog/${strArchive}"); + } + } + + # Need some sort of backup validate - create a manifest and compare the backup manifest + # !!! DO IT + + # Save the backup conf file final time + backup_manifest_save($strBackupConfFile, \%oBackupManifest); + + # Rename the backup tmp path to complete the backup + &log(DEBUG, "moving ${strBackupTmpPath} to " . $oFile->path_get(PATH_BACKUP_CLUSTER, $strBackupPath)); + $oFile->file_move(PATH_BACKUP_TMP, undef, PATH_BACKUP_CLUSTER, $strBackupPath); +} + +#################################################################################################################################### +# ARCHIVE_LIST_GET +# +# Generates a range of archive log file names given the start and end log file name. For pre-9.3 databases, use bSkipFF to exclude +# the FF that prior versions did not generate. +#################################################################################################################################### +sub archive_list_get +{ + my $strArchiveStart = shift; + my $strArchiveStop = shift; + my $bSkipFF = shift; + + # strSkipFF default to false + $bSkipFF = defined($bSkipFF) ? $bSkipFF : false; + + if ($bSkipFF) + { + &log(TRACE, "archive_list_get: pre-9.3 database, skipping log FF"); + } + else + { + &log(TRACE, "archive_list_get: post-9.3 database, including log FF"); + } + + my $strTimeline = substr($strArchiveStart, 0, 8); + my @stryArchive; + my $iArchiveIdx = 0; + + if ($strTimeline ne substr($strArchiveStop, 0, 8)) + { + confess "Timelines between ${strArchiveStart} and ${strArchiveStop} differ"; + } + + my $iStartMajor = hex substr($strArchiveStart, 8, 8); + my $iStartMinor = hex substr($strArchiveStart, 16, 8); + + my $iStopMajor = hex substr($strArchiveStop, 8, 8); + my $iStopMinor = hex substr($strArchiveStop, 16, 8); + + $stryArchive[$iArchiveIdx] = uc(sprintf("${strTimeline}%08x%08x", $iStartMajor, $iStartMinor)); + $iArchiveIdx += 1; + + while (!($iStartMajor == $iStopMajor && $iStartMinor == $iStopMinor)) + { + $iStartMinor += 1; + + if ($bSkipFF && $iStartMinor == 255 || !$bSkipFF && $iStartMinor == 256) + { + $iStartMajor += 1; + $iStartMinor = 0; + } + + $stryArchive[$iArchiveIdx] = uc(sprintf("${strTimeline}%08x%08x", $iStartMajor, $iStartMinor)); + $iArchiveIdx += 1; + } + + &log(TRACE, " archive_list_get: $strArchiveStart:$strArchiveStop (@stryArchive)"); + + return @stryArchive; +} + +#################################################################################################################################### +# BACKUP_EXPIRE +# +# Removes expired backups and archive logs from the backup directory. Partial backups are not counted for expiration, so if full +# or differential retention is set to 2, there must be three complete backups before the oldest one can be deleted. +# +# iFullRetention - Optional, must be greater than 0 when supplied. +# iDifferentialRetention - Optional, must be greater than 0 when supplied. +# strArchiveRetention - Optional, must be (full,differential/diff,incremental/incr) when supplied +# iArchiveRetention - Required when strArchiveRetention is supplied. Must be greater than 0. +#################################################################################################################################### +sub backup_expire +{ + my $strBackupClusterPath = shift; # Base path to cluster backup + my $iFullRetention = shift; # Number of full backups to keep + my $iDifferentialRetention = shift; # Number of differential backups to keep + my $strArchiveRetentionType = shift; # Type of backup to base archive retention on + my $iArchiveRetention = shift; # Number of backups worth of archive to keep + + my $strPath; + my @stryPath; + + # Find all the expired full backups + if (defined($iFullRetention)) + { + # Make sure iFullRetention is valid + if (!looks_like_number($iFullRetention) || $iFullRetention < 1) + { + confess &log(ERROR, "full_rentention must be a number >= 1"); + } + + my $iIndex = $iFullRetention; + @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); + + while (defined($stryPath[$iIndex])) + { + &log(INFO, "removed expired full backup: " . $stryPath[$iIndex]); + + # Delete all backups that depend on the full backup. Done in reverse order so that remaining backups will still + # be consistent if the process dies + foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, "^" . $stryPath[$iIndex] . ".*", "reverse")) + { + system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}"); + } + + $iIndex++; + } + } + + # Find all the expired differential backups + if (defined($iDifferentialRetention)) + { + # Make sure iDifferentialRetention is valid + if (!looks_like_number($iDifferentialRetention) || $iDifferentialRetention < 1) + { + confess &log(ERROR, "differential_rentention must be a number >= 1"); + } + + @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 0), "reverse"); + + if (defined($stryPath[$iDifferentialRetention])) + { + # Get a list of all differential and incremental backups + foreach $strPath ($oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(0, 1, 1), "reverse")) + { + # Remove all differential and incremental backups before the oldest valid differential + if (substr($strPath, 0, length($strPath) - 1) lt $stryPath[$iDifferentialRetention]) + { + system("rm -rf ${strBackupClusterPath}/${strPath}") == 0 or confess &log(ERROR, "unable to delete backup ${strPath}"); + &log(INFO, "removed expired diff/incr backup ${strPath}"); + } + } + } + } + + # If no archive retention type is set then exit + if (!defined($strArchiveRetentionType)) + { + &log(INFO, "archive rentention type not set - archive logs will not be expired"); + return; + } + + # Determine which backup type to use for archive retention (full, differential, incremental) + if ($strArchiveRetentionType eq "full") + { + @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 0, 0), "reverse"); + } + elsif ($strArchiveRetentionType eq "differential" || $strArchiveRetentionType eq "diff") + { + @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 0), "reverse"); + } + elsif ($strArchiveRetentionType eq "incremental" || $strArchiveRetentionType eq "incr") + { + @stryPath = $oFile->file_list_get(PATH_BACKUP_CLUSTER, undef, backup_regexp_get(1, 1, 1), "reverse"); + } + else + { + confess &log(ERROR, "unknown archive_retention_type '${strArchiveRetentionType}'"); + } + + # Make sure that iArchiveRetention is set and valid + if (!defined($iArchiveRetention)) + { + confess &log(ERROR, "archive_rentention must be set if archive_retention_type is set"); + return; + } + + if (!looks_like_number($iArchiveRetention) || $iArchiveRetention < 1) + { + confess &log(ERROR, "archive_rentention must be a number >= 1"); + } + + # if no backups were found then preserve current archive logs - too scary to delete them! + my $iBackupTotal = scalar @stryPath; + + if ($iBackupTotal == 0) + { + return; + } + + # See if enough backups exist for expiration to start + my $strArchiveRetentionBackup = $stryPath[$iArchiveRetention - 1]; + + if (!defined($strArchiveRetentionBackup)) + { + if ($strArchiveRetentionType eq "full" && scalar @stryPath > 0) + { + &log(INFO, "fewer than required backups for retention, but since archive_retention_type = full using oldest full backup"); + $strArchiveRetentionBackup = $stryPath[scalar @stryPath - 1]; + } + + if (!defined($strArchiveRetentionBackup)) + { + return; + } + } + + # Get the archive logs that need to be kept. To be cautious we will keep all the archive logs starting from this backup + # even though they are also in the pg_xlog directory (since they have been copied more than once). + &log(INFO, "archive retention based on backup " . $strArchiveRetentionBackup); + + my %oManifest = backup_manifest_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/$strArchiveRetentionBackup/backup.manifest"); + my $strArchiveLast = ${oManifest}{backup}{"archive-start"}; + + if (!defined($strArchiveLast)) + { + confess &log(ERROR, "invalid archive location retrieved ${strArchiveRetentionBackup}"); + } + + &log(INFO, "archive retention starts at " . $strArchiveLast); + + # Remove any archive directories or files that are out of date + foreach $strPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, undef, "^[0-F]{16}\$")) + { + &log(DEBUG, "found major archive path " . $strPath); + + # If less than first 16 characters of current archive file, then remove the directory + if ($strPath lt substr($strArchiveLast, 0, 16)) + { + my $strFullPath = $oFile->path_get(PATH_BACKUP_ARCHIVE) . "/$strPath"; + + remove_tree($strFullPath) > 0 or confess &log(ERROR, "unable to remove ${strFullPath}"); + + &log(DEBUG, "removed major archive path " . $strFullPath); + } + # If equals the first 16 characters of the current archive file, then delete individual files instead + elsif ($strPath eq substr($strArchiveLast, 0, 16)) + { + my $strSubPath; + + # Look for archive files in the archive directory + foreach $strSubPath ($oFile->file_list_get(PATH_BACKUP_ARCHIVE, $strPath, "^[0-F]{24}.*\$")) + { + # Delete if the first 24 characters less than the current archive file + if ($strSubPath lt substr($strArchiveLast, 0, 24)) + { + unlink($oFile->path_get(PATH_BACKUP_ARCHIVE, $strSubPath)) or confess &log(ERROR, "unable to remove " . $strSubPath); + &log(DEBUG, "removed expired archive file " . $strSubPath); + } + } + } + } +} + +1; \ No newline at end of file diff --git a/pg_backrest_db.pm b/pg_backrest_db.pm new file mode 100644 index 000000000..7341b4f61 --- /dev/null +++ b/pg_backrest_db.pm @@ -0,0 +1,146 @@ +#################################################################################################################################### +# DB MODULE +#################################################################################################################################### +package pg_backrest_db; + +use threads; + +use Moose; +use strict; +use warnings; +use Carp; +use Net::OpenSSH; +use File::Basename; +use IPC::System::Simple qw(capture); + +use lib dirname($0); +use pg_backrest_utility; + +# Command strings +has strCommandPsql => (is => 'bare'); # PSQL command + +# Module variables +has strDbUser => (is => 'ro'); # Database user +has strDbHost => (is => 'ro'); # Database host +has oDbSSH => (is => 'bare'); # Database SSH object +has fVersion => (is => 'ro'); # Database version + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub BUILD +{ + my $self = shift; + + # Connect SSH object if db host is defined + if (defined($self->{strDbHost}) && !defined($self->{oDbSSH})) + { + &log(TRACE, "connecting to database ssh host $self->{strDbHost}"); + + # !!! This could be improved by redirecting stderr to a file to get a better error message + $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, master_stderr_discard => true, user => $self->{strDbUser}); + $self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error); + } +} + +#################################################################################################################################### +# IS_REMOTE +# +# Determine whether database operations are remote. +#################################################################################################################################### +sub is_remote +{ + my $self = shift; + + # If the SSH object is defined then db is remote + return defined($self->{oDbSSH}) ? true : false; +} + +#################################################################################################################################### +# PSQL_EXECUTE +#################################################################################################################################### +sub psql_execute +{ + my $self = shift; + my $strScript = shift; # psql script to execute + + # Get the user-defined command for psql + my $strCommand = $self->{strCommandPsql} . " -c \"${strScript}\" postgres"; + my $strResult; + + # !!! Need to capture error output with open3 and log it + + # Run remotely + if ($self->is_remote()) + { + &log(TRACE, "psql execute: remote ${strScript}"); + + $strResult = $self->{oDbSSH}->capture($strCommand) + or confess &log(ERROR, "unable to execute remote psql command '${strCommand}'"); + } + # Else run locally + else + { + &log(TRACE, "psql execute: ${strScript}"); + $strResult = capture($strCommand) or confess &log(ERROR, "unable to execute local psql command '${strCommand}'"); + } + + return $strResult; +} + +#################################################################################################################################### +# TABLESPACE_MAP_GET - Get the mapping between oid and tablespace name +#################################################################################################################################### +sub tablespace_map_get +{ + my $self = shift; + + return data_hash_build("oid\tname\n" . $self->psql_execute( + "copy (select oid, spcname from pg_tablespace) to stdout"), "\t"); +} + +#################################################################################################################################### +# VERSION_GET +#################################################################################################################################### +sub version_get +{ + my $self = shift; + + if (defined($self->{fVersion})) + { + return $self->{fVersion}; + } + + $self->{fVersion} = + trim($self->psql_execute("copy (select (regexp_matches(split_part(version(), ' ', 2), '^[0-9]+\.[0-9]+'))[1]) to stdout")); + + &log(DEBUG, "database version is $self->{fVersion}"); + + return $self->{fVersion}; +} + +#################################################################################################################################### +# BACKUP_START +#################################################################################################################################### +sub backup_start +{ + my $self = shift; + my $strLabel = shift; + + return trim($self->psql_execute("set client_min_messages = 'warning';" . + "copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}') as xlog) to stdout")); +} + +#################################################################################################################################### +# BACKUP_STOP +#################################################################################################################################### +sub backup_stop +{ + my $self = shift; + + return trim($self->psql_execute("set client_min_messages = 'warning';" . + "copy (select pg_xlogfile_name(xlog) from pg_stop_backup() as xlog) to stdout")) +} + +no Moose; + __PACKAGE__->meta->make_immutable; \ No newline at end of file diff --git a/pg_backrest_file.pm b/pg_backrest_file.pm new file mode 100644 index 000000000..59477484f --- /dev/null +++ b/pg_backrest_file.pm @@ -0,0 +1,921 @@ +#################################################################################################################################### +# FILE MODULE +#################################################################################################################################### +package pg_backrest_file; + +use threads; + +use Moose; +use strict; +use warnings; +use Carp; +use Net::OpenSSH; +use IPC::Open3; +use File::Basename; +use IPC::System::Simple qw(capture); + +use lib dirname($0); +use pg_backrest_utility; + +use Exporter qw(import); +our @EXPORT = qw(PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE PATH_BACKUP_CLUSTER PATH_BACKUP_TMP PATH_BACKUP_ARCHIVE); + +# Extension and permissions +has strCompressExtension => (is => 'ro', default => 'gz'); +has strDefaultPathPermission => (is => 'bare', default => '0750'); +has strDefaultFilePermission => (is => 'ro', default => '0640'); + +# Command strings +has strCommandChecksum => (is => 'bare'); +has strCommandCompress => (is => 'bare'); +has strCommandDecompress => (is => 'bare'); +has strCommandCat => (is => 'bare', default => 'cat %file%'); +has strCommandManifest => (is => 'bare'); + +# Module variables +has strDbUser => (is => 'bare'); # Database user +has strDbHost => (is => 'bare'); # Database host +has oDbSSH => (is => 'bare'); # Database SSH object + +has strBackupUser => (is => 'bare'); # Backup user +has strBackupHost => (is => 'bare'); # Backup host +has oBackupSSH => (is => 'bare'); # Backup SSH object +has strBackupPath => (is => 'bare'); # Backup base path +has strBackupClusterPath => (is => 'bare'); # Backup cluster path + +# Process flags +has bNoCompression => (is => 'bare'); +has strStanza => (is => 'bare'); +has iThreadIdx => (is => 'bare'); + +#################################################################################################################################### +# CONSTRUCTOR +#################################################################################################################################### +sub BUILD +{ + my $self = shift; + + # Make sure the backup path is defined + if (!defined($self->{strBackupPath})) + { + confess &log(ERROR, "common:backup_path undefined"); + } + + # Create the backup cluster path + $self->{strBackupClusterPath} = $self->{strBackupPath} . "/" . $self->{strStanza}; + + # Create the ssh options string + if (defined($self->{strBackupHost}) || defined($self->{strDbHost})) + { + my $strOptionSSH = "Compression=no"; + + if ($self->{bNoCompression}) + { + $strOptionSSH = "Compression=yes"; + } + + # Connect SSH object if backup host is defined + if (!defined($self->{oBackupSSH}) && defined($self->{strBackupHost})) + { + &log(TRACE, "connecting to backup ssh host " . $self->{strBackupHost}); + + # !!! This could be improved by redirecting stderr to a file to get a better error message + $self->{oBackupSSH} = Net::OpenSSH->new($self->{strBackupHost}, timeout => 300, user => $self->{strBackupUser}, master_opts => [-o => $strOptionSSH]); + $self->{oBackupSSH}->error and confess &log(ERROR, "unable to connect to $self->{strBackupHost}: " . $self->{oBackupSSH}->error); + } + + # Connect SSH object if db host is defined + if (!defined($self->{oDbSSH}) && defined($self->{strDbHost})) + { + &log(TRACE, "connecting to database ssh host $self->{strDbHost}"); + + # !!! This could be improved by redirecting stderr to a file to get a better error message + $self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, timeout => 300, user => $self->{strDbUser}, master_opts => [-o => $strOptionSSH]); + $self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error); + } + } +} + +#################################################################################################################################### +# CLONE +#################################################################################################################################### +sub clone +{ + my $self = shift; + my $iThreadIdx = shift; + + return pg_backrest_file->new + ( + strCompressExtension => $self->{strCompressExtension}, + strDefaultPathPermission => $self->{strDefaultPathPermission}, + strDefaultFilePermission => $self->{strDefaultFilePermission}, + strCommandChecksum => $self->{strCommandChecksum}, + strCommandCompress => $self->{strCommandCompress}, + strCommandDecompress => $self->{strCommandDecompress}, + strCommandCat => $self->{strCommandCat}, + strCommandManifest => $self->{strCommandManifest}, +# oDbSSH => $self->{strDbSSH}, + strDbUser => $self->{strDbUser}, + strDbHost => $self->{strDbHost}, +# oBackupSSH => $self->{strBackupSSH}, + strBackupUser => $self->{strBackupUser}, + strBackupHost => $self->{strBackupHost}, + strBackupPath => $self->{strBackupPath}, + strBackupClusterPath => $self->{strBackupClusterPath}, + bNoCompression => $self->{bNoCompression}, + strStanza => $self->{strStanza}, + iThreadIdx => $iThreadIdx + ); +} + +#################################################################################################################################### +# PATH_GET +#################################################################################################################################### +use constant +{ + PATH_DB => 'db', + PATH_DB_ABSOLUTE => 'db:absolute', + PATH_BACKUP => 'backup', + PATH_BACKUP_ABSOLUTE => 'backup:absolute', + PATH_BACKUP_CLUSTER => 'backup:cluster', + PATH_BACKUP_TMP => 'backup:tmp', + PATH_BACKUP_ARCHIVE => 'backup:archive' +}; + +sub path_type_get +{ + my $self = shift; + my $strType = shift; + + # If db type + if ($strType =~ /^db(\:.*){0,1}/) + { + return PATH_DB; + } + # Else if backup type + elsif ($strType =~ /^backup(\:.*){0,1}/) + { + return PATH_BACKUP; + } + + # Error when path type not recognized + confess &log(ASSERT, "no known path types in '${strType}'"); +} + +sub path_get +{ + my $self = shift; + my $strType = shift; # Base type of the path to get (PATH_DB_ABSOLUTE, PATH_BACKUP_TMP, etc) + my $strFile = shift; # File to append to the base path (can include a path as well) + my $bTemp = shift; # Return the temp file for this path type - only some types have temp files + + # Only allow temp files for PATH_BACKUP_ARCHIVE and PATH_BACKUP_TMP + if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP)) + { + confess &log(ASSERT, "temp file not supported on path " . $strType); + } + + # Get absolute db path + if ($strType eq PATH_DB_ABSOLUTE) + { + return $strFile; + } + + # Make sure the base backup path is defined + if (!defined($self->{strBackupPath})) + { + confess &log(ASSERT, "\$strBackupPath not yet defined"); + } + + # Get absolute backup path + if ($strType eq PATH_BACKUP_ABSOLUTE) + { + # Need a check in here to make sure this is relative to the backup path + + return $strFile; + } + + # Get base backup path + if ($strType eq PATH_BACKUP) + { + return $self->{strBackupPath} . (defined($strFile) ? "/${strFile}" : ""); + } + + # Make sure the cluster is defined + if (!defined($self->{strStanza})) + { + confess &log(ASSERT, "\$strStanza not yet defined"); + } + + # Get the backup tmp path + if ($strType eq PATH_BACKUP_TMP) + { + my $strTempPath = "$self->{strBackupPath}/temp/$self->{strStanza}.tmp"; + + if (defined($bTemp) && $bTemp) + { + return "${strTempPath}/file.tmp" . (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : ""); + } + + return "${strTempPath}" . (defined($strFile) ? "/${strFile}" : ""); + } + + # Get the backup archive path + if ($strType eq PATH_BACKUP_ARCHIVE) + { + my $strArchivePath = "$self->{strBackupPath}/archive/$self->{strStanza}"; + my $strArchive; + + if (defined($bTemp) && $bTemp) + { + return "${strArchivePath}/file.tmp" . (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : ""); + } + + if (defined($strFile)) + { + $strArchive = substr(basename($strFile), 0, 24); + + if ($strArchive !~ /^([0-F]){24}$/) + { + return "${strArchivePath}/${strFile}"; + } + } + + return $strArchivePath . (defined($strArchive) ? "/" . substr($strArchive, 0, 16) : "") . + (defined($strFile) ? "/" . $strFile : ""); + } + + if ($strType eq PATH_BACKUP_CLUSTER) + { + return $self->{strBackupPath} . "/backup/$self->{strStanza}" . (defined($strFile) ? "/${strFile}" : ""); + } + + # Error when path type not recognized + confess &log(ASSERT, "no known path types in '${strType}'"); +} + +#################################################################################################################################### +# LINK_CREATE +#################################################################################################################################### +sub link_create +{ + my $self = shift; + my $strSourcePathType = shift; + my $strSourceFile = shift; + my $strDestinationPathType = shift; + my $strDestinationFile = shift; + my $bHard = shift; + my $bRelative = shift; + my $bPathCreate = shift; + + # if bHard is not defined default to false + $bHard = defined($bHard) ? $bHard : false; + + # if bRelative is not defined or bHard is true, default to false + $bRelative = !defined($bRelative) || $bHard ? false : $bRelative; + + # if bPathCreate is not defined, default to true + $bPathCreate = defined($bPathCreate) ? $bPathCreate : true; + + # Source and destination path types must be the same (both PATH_DB or both PATH_BACKUP) + if ($self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType)) + { + confess &log(ASSERT, "path types must be equal in link create"); + } + + # Generate source and destination files + my $strSource = $self->path_get($strSourcePathType, $strSourceFile); + my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile); + + # If the destination path is backup and does not exist, create it + if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP) + { + $self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination)); + } + + unless (-e $strSource) + { + if (-e $strSource . ".$self->{strCompressExtension}") + { + $strSource .= ".$self->{strCompressExtension}"; + $strDestination .= ".$self->{strCompressExtension}"; + } + else + { + # Error when a hardlink will be created on a missing file + if ($bHard) + { + confess &log(ASSERT, "unable to find ${strSource}(.$self->{strCompressExtension}) for link"); + } + } + } + + # Generate relative path if requested + if ($bRelative) + { + my $iCommonLen = common_prefix($strSource, $strDestination); + + if ($iCommonLen != 0) + { + $strSource = ("../" x substr($strDestination, $iCommonLen) =~ tr/\///) . substr($strSource, $iCommonLen); + } + } + + # Create the command + my $strCommand = "ln" . (!$bHard ? " -s" : "") . " ${strSource} ${strDestination}"; + + # Run remotely + if ($self->is_remote($strSourcePathType)) + { + &log(TRACE, "link_create: remote ${strSourcePathType} '${strCommand}'"); + + my $oSSH = $self->remote_get($strSourcePathType); + $oSSH->system($strCommand) or confess &log("unable to create link from ${strSource} to ${strDestination}"); + } + # Run locally + else + { + &log(TRACE, "link_create: local '${strCommand}'"); + system($strCommand) == 0 or confess &log("unable to create link from ${strSource} to ${strDestination}"); + } +} + +#################################################################################################################################### +# PATH_CREATE +# +# Creates a path locally or remotely. Currently does not error if the path already exists. Also does not set permissions if the +# path aleady exists. +#################################################################################################################################### +sub path_create +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + my $strPermission = shift; + + # If no permissions are given then use the default + if (!defined($strPermission)) + { + $strPermission = $self->{strDefaultPathPermission}; + } + + # Get the path to create + my $strPathCreate = $strPath; + + if (defined($strPathType)) + { + $strPathCreate = $self->path_get($strPathType, $strPath); + } + + my $strCommand = "mkdir -p -m ${strPermission} ${strPathCreate}"; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "path_create: remote ${strPathType} '${strCommand}'"); + + my $oSSH = $self->remote_get($strPathType); + $oSSH->system($strCommand) or confess &log("unable to create remote path ${strPathType}:${strPath}"); + } + # Run locally + else + { + &log(TRACE, "path_create: local '${strCommand}'"); + system($strCommand) == 0 or confess &log(ERROR, "unable to create path ${strPath}"); + } +} + +#################################################################################################################################### +# IS_REMOTE +# +# Determine whether any operations are being performed remotely. If $strPathType is defined, the function will return true if that +# path is remote. If $strPathType is not defined, then function will return true if any path is remote. +#################################################################################################################################### +sub is_remote +{ + my $self = shift; + my $strPathType = shift; + + # If the SSH object is defined then some paths are remote + if (defined($self->{oDbSSH}) || defined($self->{oBackupSSH})) + { + # If path type is not defined but the SSH object is, then some paths are remote + if (!defined($strPathType)) + { + return true; + } + + # If a host is defined for the path then it is remote + if (defined($self->{strBackupHost}) && $self->path_type_get($strPathType) eq PATH_BACKUP || + defined($self->{strDbHost}) && $self->path_type_get($strPathType) eq PATH_DB) + { + return true; + } + } + + return false; +} + +#################################################################################################################################### +# REMOTE_GET +# +# Get remote SSH object depending on the path type. +#################################################################################################################################### +sub remote_get +{ + my $self = shift; + my $strPathType = shift; + + # Get the db SSH object + if ($self->path_type_get($strPathType) eq PATH_DB && defined($self->{oDbSSH})) + { + return $self->{oDbSSH}; + } + + # Get the backup SSH object + if ($self->path_type_get($strPathType) eq PATH_BACKUP && defined($self->{oBackupSSH})) + { + return $self->{oBackupSSH} + } + + # Error when no ssh object is found + confess &log(ASSERT, "path type ${strPathType} does not have a defined ssh object"); +} + +#################################################################################################################################### +# FILE_MOVE +# +# Moves a file locally or remotely. +#################################################################################################################################### +sub file_move +{ + my $self = shift; + my $strSourcePathType = shift; + my $strSourceFile = shift; + my $strDestinationPathType = shift; + my $strDestinationFile = shift; + my $bPathCreate = shift; + + # if bPathCreate is not defined, default to true + $bPathCreate = defined($bPathCreate) ? $bPathCreate : true; + + &log(TRACE, "file_move: ${strSourcePathType}: " . (defined($strSourceFile) ? ":${strSourceFile}" : "") . + " to ${strDestinationPathType}" . (defined($strDestinationFile) ? ":${strDestinationFile}" : "")); + + # Get source and desination files + if ($self->path_type_get($strSourcePathType) ne $self->path_type_get($strSourcePathType)) + { + confess &log(ASSERT, "source and destination path types must be equal"); + } + + my $strSource = $self->path_get($strSourcePathType, $strSourceFile); + my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile); + + # If the destination path is backup and does not exist, create it + if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP) + { + $self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination)); + } + + my $strCommand = "mv ${strSource} ${strDestination}"; + + # Run remotely + if ($self->is_remote($strDestinationPathType)) + { + &log(TRACE, "file_move: remote ${strDestinationPathType} '${strCommand}'"); + + my $oSSH = $self->remote_get($strDestinationPathType); + $oSSH->system($strCommand) + or confess &log("unable to move remote ${strDestinationPathType}:${strSourceFile} to ${strDestinationFile}"); + } + # Run locally + else + { + &log(TRACE, "file_move: '${strCommand}'"); + + system($strCommand) == 0 or confess &log("unable to move local ${strSourceFile} to ${strDestinationFile}"); + } +} + +#################################################################################################################################### +# FILE_COPY +#################################################################################################################################### +sub file_copy +{ + my $self = shift; + my $strSourcePathType = shift; + my $strSourceFile = shift; + my $strDestinationPathType = shift; + my $strDestinationFile = shift; + my $bNoCompressionOverride = shift; + my $lModificationTime = shift; + my $strPermission = shift; + my $bPathCreate = shift; + my $bConfessCopyError = shift; + + # if bPathCreate is not defined, default to true + $bPathCreate = defined($bPathCreate) ? $bPathCreate : true; + $bConfessCopyError = defined($bConfessCopyError) ? $bConfessCopyError : false; + + &log(TRACE, "file_copy: ${strSourcePathType}: " . (defined($strSourceFile) ? ":${strSourceFile}" : "") . + " to ${strDestinationPathType}" . (defined($strDestinationFile) ? ":${strDestinationFile}" : "")); + + # Modification time and permissions cannot be set remotely + if ((defined($lModificationTime) || defined($strPermission)) && $self->is_remote($strDestinationPathType)) + { + confess &log(ASSERT, "modification time and permissions cannot be set on remote destination file"); + } + + # Generate source, destination and tmp filenames + my $strSource = $self->path_get($strSourcePathType, $strSourceFile); + my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile); + my $strDestinationTmp = $self->path_get($strDestinationPathType, $strDestinationFile, true); + + # Is this already a compressed file? + my $bAlreadyCompressed = $strSource =~ "^.*\.$self->{strCompressExtension}\$"; + + if ($bAlreadyCompressed && $strDestination !~ "^.*\.$self->{strCompressExtension}\$") + { + $strDestination .= ".$self->{strCompressExtension}"; + } + + # Does the file need compression? + my $bCompress = !((defined($bNoCompressionOverride) && $bNoCompressionOverride) || + (!defined($bNoCompressionOverride) && $self->{bNoCompression})); + + # If the destination path is backup and does not exist, create it + if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP) + { + $self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination)); + } + + # Generate the command string depending on compression/decompression/cat + my $strCommand = $self->{strCommandCat}; + + if (!$bAlreadyCompressed && $bCompress) + { + $strCommand = $self->{strCommandCompress}; + $strDestination .= ".gz"; + } + elsif ($bAlreadyCompressed && !$bCompress) + { + $strCommand = $self->{strCommandDecompress}; + $strDestination = substr($strDestination, 0, length($strDestination) - length($self->{strCompressExtension}) - 1); + } + + $strCommand =~ s/\%file\%/${strSource}/g; + $strCommand .= " 2> /dev/null"; + + # If this command is remote on only one side + if ($self->is_remote($strSourcePathType) && !$self->is_remote($strDestinationPathType) || + !$self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType)) + { + # Else if the source is remote + if ($self->is_remote($strSourcePathType)) + { + &log(TRACE, "file_copy: remote ${strSource} to local ${strDestination}"); + + # Open the destination file for writing (will be streamed from the ssh session) + my $hFile; + open($hFile, ">", $strDestinationTmp) or confess &log(ERROR, "cannot open ${strDestination}"); + + # Execute the command through ssh + my $oSSH = $self->remote_get($strSourcePathType); + + unless ($oSSH->system({stdout_fh => $hFile}, $strCommand)) + { + close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}"); + + my $strResult = "unable to execute ssh '${strCommand}'"; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } + + # Close the destination file handle + close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}"); + } + # Else if the destination is remote + elsif ($self->is_remote($strDestinationPathType)) + { + &log(TRACE, "file_copy: local ${strSource} ($strCommand) to remote ${strDestination}"); + + # Open the input command as a stream + my $hOut; + my $pId = open3(undef, $hOut, undef, $strCommand) or confess(ERROR, "unable to execute '${strCommand}'"); + + # Execute the command though ssh + my $oSSH = $self->remote_get($strDestinationPathType); + $oSSH->system({stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'"); + + # Wait for the stream process to finish + waitpid($pId, 0); + my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8; + + if ($iExitStatus != 0) + { + my $strResult = "command '${strCommand}' returned " . $iExitStatus; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } + } + } + # If the source and destination are both remote but not the same remote + elsif ($self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType) && + $self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType)) + { + &log(TRACE, "file_copy: remote ${strSource} to remote ${strDestination}"); + confess &log(ASSERT, "remote source and destination not supported"); + } + # Else this is a local command or remote where both sides are the same remote + else + { + # Complete the command by redirecting to the destination tmp file + $strCommand .= " > ${strDestinationTmp}"; + + if ($self->is_remote($strSourcePathType)) + { + &log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'"); + + my $oSSH = $self->remote_get($strSourcePathType); + + unless($oSSH->system($strCommand)) + { + my $strResult = "unable to execute remote command ${strCommand}:" . oSSH->error; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } + } + else + { + &log(TRACE, "file_copy: local '${strCommand}'"); + + unless(system($strCommand) == 0) + { + my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}"; + $bConfessCopyError ? confess &log(ERROR, $strResult) : return false; + } + } + } + + # Set the file permission if required (this only works locally for now) + if (defined($strPermission)) + { + &log(TRACE, "file_copy: chmod ${strPermission}"); + + system("chmod ${strPermission} ${strDestinationTmp}") == 0 + or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmp}"); + } + + # Set the file modification time if required (this only works locally for now) + if (defined($lModificationTime)) + { + &log(TRACE, "file_copy: time ${lModificationTime}"); + + utime($lModificationTime, $lModificationTime, $strDestinationTmp) + or confess &log(ERROR, "unable to set time for local ${strDestinationTmp}"); + } + + # Move the file from tmp to final destination + $self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp, + $self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate); + + return true; +} + +#################################################################################################################################### +# FILE_HASH_GET +#################################################################################################################################### +sub file_hash_get +{ + my $self = shift; + my $strPathType = shift; + my $strFile = shift; + + # For now this operation is not supported remotely. Not currently needed. + if ($self->is_remote($strPathType)) + { + confess &log(ASSERT, "remote operation not supported"); + } + + if (!defined($self->{strCommandChecksum})) + { + confess &log(ASSERT, "\$strCommandChecksum not defined"); + } + + my $strPath = $self->path_get($strPathType, $strFile); + my $strCommand; + + if (-e $strPath) + { + $strCommand = $self->{strCommandChecksum}; + $strCommand =~ s/\%file\%/${strPath}/g; + } + elsif (-e $strPath . ".$self->{strCompressExtension}") + { + $strCommand = $self->{strCommandDecompress}; + $strCommand =~ s/\%file\%/${strPath}/g; + $strCommand .= " | " . $self->{strCommandChecksum}; + $strCommand =~ s/\%file\%//g; + } + else + { + confess &log(ASSERT, "unable to find $strPath(.$self->{strCompressExtension}) for checksum"); + } + + return trim(capture($strCommand)) or confess &log(ERROR, "unable to checksum ${strPath}"); +} +#################################################################################################################################### +# FILE_COMPRESS +#################################################################################################################################### +sub file_compress +{ + my $self = shift; + my $strPathType = shift; + my $strFile = shift; + + # For now this operation is not supported remotely. Not currently needed. + if ($self->is_remote($strPathType)) + { + confess &log(ASSERT, "remote operation not supported"); + } + + if (!defined($self->{strCommandCompress})) + { + confess &log(ASSERT, "\$strCommandChecksum not defined"); + } + + my $strPath = $self->path_get($strPathType, $strFile); + + # Build the command + my $strCommand = $self->{strCommandCompress}; + $strCommand =~ s/\%file\%/${strPath}/g; + $strCommand =~ s/\ \-\-stdout//g; + + system($strCommand) == 0 or confess &log(ERROR, "unable to compress ${strPath}: ${strCommand}"); +} + +#################################################################################################################################### +# FILE_LIST_GET +#################################################################################################################################### +sub file_list_get +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + my $strExpression = shift; + my $strSortOrder = shift; + + # For now this operation is not supported remotely. Not currently needed. + if ($self->is_remote($strPathType)) + { + confess &log(ASSERT, "remote operation not supported"); + } + + my $strPathList = $self->path_get($strPathType, $strPath); + my $hDir; + + opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}"); + my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}"); + close $hDir; + + my @stryFile; + + if (@stryFileAll) + { + @stryFile = grep(/$strExpression/i, @stryFileAll) + } + + if (@stryFile) + { + if (defined($strSortOrder) && $strSortOrder eq "reverse") + { + return sort {$b cmp $a} @stryFile; + } + else + { + return sort @stryFile; + } + } + + return @stryFile; +} + +#################################################################################################################################### +# FILE_EXISTS +#################################################################################################################################### +sub file_exists +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + + # Get the root path for the manifest + my $strPathExists = $self->path_get($strPathType, $strPath); + + # Builds the exists command + my $strCommand = "ls ${strPathExists} 2> /dev/null"; + + # Run the file exists command + my $strExists = ""; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}"); + + my $oSSH = $self->remote_get($strPathType); + $strExists = $oSSH->capture($strCommand); + } + # Run locally + else + { + &log(TRACE, "file_exists: local ${strPathType}:${strPathExists}"); + $strExists = capture($strCommand); + } + + # If the return from ls eq strPathExists then true + return ($strExists eq $strPathExists); +} + +#################################################################################################################################### +# FILE_REMOVE +#################################################################################################################################### +sub file_remove +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + my $bTemp = shift; + my $bErrorIfNotExists = shift; + + if (!defined($bErrorIfNotExists)) + { + $bErrorIfNotExists = false; + } + + # Get the root path for the manifest + my $strPathRemove = $self->path_get($strPathType, $strPath, $bTemp); + + # Builds the exists command + my $strCommand = "rm -f ${strPathRemove}"; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "file_remove: remote ${strPathType}:${strPathRemove}"); + + my $oSSH = $self->remote_get($strPathType); + $oSSH->system($strCommand) or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove remote ${strPathType}:${strPathRemove}") : true; + } + # Run locally + else + { + &log(TRACE, "file_exists: local ${strPathType}:${strPathRemove}"); + system($strCommand) == 0 or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove local ${strPathType}:${strPathRemove}") : true; + } +} + +#################################################################################################################################### +# MANIFEST_GET +# +# Builds a path/file manifest starting with the base path and including all subpaths. The manifest contains all the information +# needed to perform a backup or a delta with a previous backup. +#################################################################################################################################### +sub manifest_get +{ + my $self = shift; + my $strPathType = shift; + my $strPath = shift; + + &log(TRACE, "manifest: " . $self->{strCommandManifest}); + + # Get the root path for the manifest + my $strPathManifest = $self->path_get($strPathType, $strPath); + + # Builds the manifest command + my $strCommand = $self->{strCommandManifest}; + $strCommand =~ s/\%path\%/${strPathManifest}/g; + $strCommand .= " 2> /dev/null"; + + # Run the manifest command + my $strManifest; + + # Run remotely + if ($self->is_remote($strPathType)) + { + &log(TRACE, "manifest_get: remote ${strPathType}:${strPathManifest}"); + + my $oSSH = $self->remote_get($strPathType); + $strManifest = $oSSH->capture($strCommand) or confess &log(ERROR, "unable to execute remote command '${strCommand}'"); + } + # Run locally + else + { + &log(TRACE, "manifest_get: local ${strPathType}:${strPathManifest}"); + $strManifest = capture($strCommand) or confess &log(ERROR, "unable to execute local command '${strCommand}'"); + } + + # Load the manifest into a hash + return data_hash_build("name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination\n" . + $strManifest, "\t", "."); +} + +no Moose; + __PACKAGE__->meta->make_immutable; \ No newline at end of file diff --git a/pg_backrest_utility.pm b/pg_backrest_utility.pm new file mode 100644 index 000000000..430c33708 --- /dev/null +++ b/pg_backrest_utility.pm @@ -0,0 +1,354 @@ +#################################################################################################################################### +# UTILITY MODULE +#################################################################################################################################### +package pg_backrest_utility; + +use threads; + +use strict; +use warnings; +use Carp; +use IPC::System::Simple qw(capture); +use Fcntl qw(:DEFAULT :flock); + +use Exporter qw(import); + +our @EXPORT = qw(data_hash_build trim common_prefix wait_for_file date_string_get file_size_format execute + log log_file_set log_level_set + lock_file_create lock_file_remove + TRACE DEBUG ERROR ASSERT WARN INFO true false); + +# Global constants +use constant +{ + true => 1, + false => 0 +}; + +use constant +{ + TRACE => 'TRACE', + DEBUG => 'DEBUG', + INFO => 'INFO', + WARN => 'WARN', + ERROR => 'ERROR', + ASSERT => 'ASSERT', + OFF => 'OFF' +}; + +my $hLogFile; +my $strLogLevelFile = ERROR; +my $strLogLevelConsole = ERROR; +my %oLogLevelRank; + +my $strLockFile; +my $hLockFile; + +$oLogLevelRank{TRACE}{rank} = 6; +$oLogLevelRank{DEBUG}{rank} = 5; +$oLogLevelRank{INFO}{rank} = 4; +$oLogLevelRank{WARN}{rank} = 3; +$oLogLevelRank{ERROR}{rank} = 2; +$oLogLevelRank{ASSERT}{rank} = 1; +$oLogLevelRank{OFF}{rank} = 0; + +#################################################################################################################################### +# LOCK_FILE_CREATE +#################################################################################################################################### +sub lock_file_create +{ + my $strLockFileParam = shift; + + $strLockFile = $strLockFileParam; + + if (defined($hLockFile)) + { + confess &lock(ASSERT, "${strLockFile} lock is already held, cannot create lock ${strLockFile}"); + } + + sysopen($hLockFile, $strLockFile, O_WRONLY | O_CREAT) + or confess &log(ERROR, "unable to open lock file ${strLockFile}"); + + if (!flock($hLockFile, LOCK_EX | LOCK_NB)) + { + close($hLockFile); + return 0; + } + + return $hLockFile; +} + +#################################################################################################################################### +# LOCK_FILE_REMOVE +#################################################################################################################################### +sub lock_file_remove +{ + if (defined($hLockFile)) + { + close($hLockFile); + unlink($strLockFile) or confess &log(ERROR, "unable to remove lock file ${strLockFile}"); + + $hLockFile = undef; + $strLockFile = undef; + } + else + { + confess &log(ASSERT, "there is no lock to free"); + } +} + +#################################################################################################################################### +# DATA_HASH_BUILD - Hash a delimited file with header +#################################################################################################################################### +sub data_hash_build +{ + my $strData = shift; + my $strDelimiter = shift; + my $strUndefinedKey = shift; + + my @stryFile = split("\n", $strData); + my @stryHeader = split($strDelimiter, $stryFile[0]); + + my %oHash; + + for (my $iLineIdx = 1; $iLineIdx < scalar @stryFile; $iLineIdx++) + { + my @stryLine = split($strDelimiter, $stryFile[$iLineIdx]); + + if (!defined($stryLine[0]) || $stryLine[0] eq "") + { + $stryLine[0] = $strUndefinedKey; + } + + for (my $iColumnIdx = 1; $iColumnIdx < scalar @stryHeader; $iColumnIdx++) + { + if (defined($oHash{"$stryHeader[0]"}{"$stryLine[0]"}{"$stryHeader[$iColumnIdx]"})) + { + confess "the first column must be unique to build the hash"; + } + + $oHash{"$stryHeader[0]"}{"$stryLine[0]"}{"$stryHeader[$iColumnIdx]"} = $stryLine[$iColumnIdx]; + } + } + + return %oHash; +} + +#################################################################################################################################### +# TRIM - trim whitespace off strings +#################################################################################################################################### +sub trim +{ + my $strBuffer = shift; + + $strBuffer =~ s/^\s+|\s+$//g; + + return $strBuffer; +} + +#################################################################################################################################### +# WAIT_FOR_FILE +#################################################################################################################################### +sub wait_for_file +{ + my $strDir = shift; + my $strRegEx = shift; + my $iSeconds = shift; + + my $lTime = time(); + my $hDir; + + while ($lTime > time() - $iSeconds) + { + opendir $hDir, $strDir or die "Could not open dir: $!\n"; + my @stryFile = grep(/$strRegEx/i, readdir $hDir); + close $hDir; + + if (scalar @stryFile == 1) + { + return; + } + + sleep(1); + } + + confess &log(ERROR, "could not find $strDir/$strRegEx after $iSeconds second(s)"); +} + +#################################################################################################################################### +# COMMON_PREFIX +#################################################################################################################################### +sub common_prefix +{ + my $strString1 = shift; + my $strString2 = shift; + + my $iCommonLen = 0; + my $iCompareLen = length($strString1) < length($strString2) ? length($strString1) : length($strString2); + + for (my $iIndex = 0; $iIndex < $iCompareLen; $iIndex++) + { + if (substr($strString1, $iIndex, 1) ne substr($strString2, $iIndex, 1)) + { + last; + } + + $iCommonLen ++; + } + + return $iCommonLen; +} + +#################################################################################################################################### +# FILE_SIZE_FORMAT - Format file sizes in human-readable form +#################################################################################################################################### +sub file_size_format +{ + my $lFileSize = shift; + + if ($lFileSize < 1024) + { + return $lFileSize . "B"; + } + + if ($lFileSize < (1024 * 1024)) + { + return int($lFileSize / 1024) . "KB"; + } + + if ($lFileSize < (1024 * 1024 * 1024)) + { + return int($lFileSize / 1024 / 1024) . "MB"; + } + + return int($lFileSize / 1024 / 1024 / 1024) . "GB"; +} + +#################################################################################################################################### +# DATE_STRING_GET - Get the date and time string +#################################################################################################################################### +sub date_string_get +{ + my $strFormat = shift; + + if (!defined($strFormat)) + { + $strFormat = "%4d%02d%02d-%02d%02d%02d"; + } + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + + return(sprintf($strFormat, $year+1900, $mon+1, $mday, $hour, $min, $sec)); +} + +#################################################################################################################################### +# LOG_FILE_SET - set the file messages will be logged to +#################################################################################################################################### +sub log_file_set +{ + my $strFile = shift; + + $strFile .= "-" . date_string_get("%4d%02d%02d") . ".log"; + my $bExists = false; + + if (-e $strFile) + { + $bExists = true; + } + + open($hLogFile, '>>', $strFile) or confess "unable to open log file ${strFile}"; + + if ($bExists) + { + print $hLogFile "\n"; + } + + print $hLogFile "-------------------PROCESS START-------------------\n"; +} + +#################################################################################################################################### +# LOG_LEVEL_SET - set the log level for file and console +#################################################################################################################################### +sub log_level_set +{ + my $strLevelFileParam = shift; + my $strLevelConsoleParam = shift; + + if (!defined($oLogLevelRank{"${strLevelFileParam}"}{rank})) + { + confess &log(ERROR, "file log level ${strLevelFileParam} does not exist"); + } + + if (!defined($oLogLevelRank{"${strLevelConsoleParam}"}{rank})) + { + confess &log(ERROR, "console log level ${strLevelConsoleParam} does not exist"); + } + + $strLogLevelFile = $strLevelFileParam; + $strLogLevelConsole = $strLevelConsoleParam; +} + +#################################################################################################################################### +# LOG - log messages +#################################################################################################################################### +sub log +{ + my $strLevel = shift; + my $strMessage = shift; + + if (!defined($oLogLevelRank{"${strLevel}"}{rank})) + { + confess &log(ASSERT, "log level ${strLevel} does not exist"); + } + + my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time); + + if (!defined($strMessage)) + { + $strMessage = "(undefined)"; + } + + if ($strLevel eq "TRACE") + { + $strMessage = " " . $strMessage; + } + elsif ($strLevel eq "DEBUG") + { + $strMessage = " " . $strMessage; + } + + $strMessage = sprintf("%4d-%02d-%02d %02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec) . + (" " x (7 - length($strLevel))) . "${strLevel} " . (" " x (2 - length(threads->tid()))) . + threads->tid() . ": ${strMessage}\n"; + + if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank}) + { + print $strMessage; + } + + if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelFile}"}{rank}) + { + if (defined($hLogFile)) + { + print $hLogFile $strMessage; + } + } + + return $strMessage; +} + +#################################################################################################################################### +# EXECUTE - execute a command +#################################################################################################################################### +sub execute +{ + my $strCommand = shift; + my $strOutput; + +# print("$strCommand"); + $strOutput = capture($strCommand) or confess &log(ERROR, "unable to execute command ${strCommand}: " . $_); + + return $strOutput; +} + +1; \ No newline at end of file diff --git a/test/test.pl b/test/test.pl new file mode 100755 index 000000000..f0558d5ca --- /dev/null +++ b/test/test.pl @@ -0,0 +1,244 @@ +#!/usr/bin/perl + +# /Library/PostgreSQL/9.3/bin/pg_ctl start -o "-c port=7000" -D /Users/backrest/test/backup/db/20140205-103801F/base -l /Users/backrest/test/backup/db/20140205-103801F/base/postgresql.log -w -s + +#use strict; +use DBI; +use IPC::System::Simple qw(capture); +use Config::IniFiles; +use File::Find; + +sub trim +{ + local($strBuffer) = @_; + + $strBuffer =~ s/^\s+|\s+$//g; + + return $strBuffer; +} + +sub execute +{ + local($strCommand) = @_; + my $strOutput; + + print("$strCommand"); + $strOutput = trim(capture($strCommand)); + + if ($strOutput eq "") + { + print(" ... complete\n\n"); + } + else + { + print(" ... complete\n$strOutput\n\n"); + } + + return $strOutput; +} + +sub pg_create +{ + local($strPgBinPath, $strTestPath, $strTestDir, $strArchiveDir, $strBackupDir) = @_; + + execute("mkdir $strTestPath"); + execute("mkdir $strTestPath/$strTestDir"); + execute("mkdir $strTestPath/$strTestDir/ts1"); + execute("mkdir $strTestPath/$strTestDir/ts2"); + execute($strPgBinPath . "/initdb -D $strTestPath/$strTestDir/common -A trust -k"); + execute("mkdir $strTestPath/$strBackupDir"); +# execute("mkdir -p $strTestPath/$strArchiveDir"); +} + +sub pg_start +{ + local($strPgBinPath, $strDbPath, $strPort, $strAchiveCommand) = @_; + my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s"; + + execute($strCommand); +} + +sub pg_password_set +{ + local($strPgBinPath, $strPath, $strUser, $strPort) = @_; + my $strCommand = "$strPgBinPath/psql --port=$strPort -c \"alter user $strUser with password 'password'\" postgres"; + + execute($strCommand); +} + +sub pg_stop +{ + local($strPgBinPath, $strPath) = @_; + my $strCommand = "$strPgBinPath/pg_ctl stop -D $strPath -w -s -m fast"; + + execute($strCommand); +} + +sub pg_drop +{ + local($strTestPath) = @_; + my $strCommand = "rm -rf $strTestPath"; + + execute($strCommand); +} + +sub pg_execute +{ + local($dbh, $strSql) = @_; + + print($strSql); + $sth = $dbh->prepare($strSql); + $sth->execute() or die; + $sth->finish(); + + print(" ... complete\n\n"); +} + +sub archive_command_build +{ + my $strBackRestBinPath = shift; + my $strDestinationPath = shift; + my $bCompression = shift; + my $bChecksum = shift; + + my $strCommand = "$strBackRestBinPath/pg_backrest.pl --stanza=db --config=$strBackRestBinPath/pg_backrest.conf"; + +# if (!$bCompression) +# { +# $strCommand .= " --no-compression" +# } +# +# if (!$bChecksum) +# { +# $strCommand .= " --no-checksum" +# } + + return $strCommand . " archive-push %p"; +} + +sub wait_for_file +{ + my $strDir = shift; + my $strRegEx = shift; + my $iSeconds = shift; + + my $lTime = time(); + my $hDir; + + while ($lTime > time() - $iSeconds) + { + opendir $hDir, $strDir or die "Could not open dir: $!\n"; + my @stryFile = grep(/$strRegEx/i, readdir $hDir); + close $hDir; + + if (scalar @stryFile == 1) + { + return; + } + + sleep(1); + } + + die "could not find $strDir/$strRegEx after $iSeconds second(s)"; +} + +sub pgbr_backup +{ + my $strBackRestBinPath = shift; + my $strCluster = shift; + + my $strCommand = "$strBackRestBinPath/pg_backrest.pl --config=$strBackRestBinPath/pg_backrest.conf backup $strCluster"; + + execute($strCommand); +} + +my $strUser = execute('whoami'); + +my $strTestPath = "/Users/dsteele/test"; +my $strDbDir = "db"; +my $strArchiveDir = "backup/db/archive"; +my $strBackupDir = "backup"; + +my $strPgBinPath = "/Library/PostgreSQL/9.3/bin"; +my $strPort = "6001"; + +my $strBackRestBinPath = "/Users/dsteele/pg_backrest"; +my $strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 0, 0); + +################################################################################ +# Stop the current test cluster if it is running and create a new one +################################################################################ +eval {pg_stop($strPgBinPath, "$strTestPath/$strDbDir")}; + +if ($@) +{ + print(" ... unable to stop pg server (ignoring): " . trim($@) . "\n\n") +} + +pg_drop($strTestPath); +pg_create($strPgBinPath, $strTestPath, $strDbDir, $strArchiveDir, $strBackupDir); +pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand); +pg_password_set($strPgBinPath, "$strTestPath/$strDbDir/common", $strUser, $strPort); + +################################################################################ +# Connect and start tests +################################################################################ +$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser, + 'password', {AutoCommit => 1}); + +pg_execute($dbh, "create tablespace ts1 location '$strTestPath/$strDbDir/ts1'"); +pg_execute($dbh, "create tablespace ts2 location '$strTestPath/$strDbDir/ts2'"); + +pg_execute($dbh, "create table test (id int)"); +pg_execute($dbh, "create table test_ts1 (id int) tablespace ts1"); +pg_execute($dbh, "create table test_ts2 (id int) tablespace ts1"); + +pg_execute($dbh, "insert into test values (1)"); +pg_execute($dbh, "select pg_switch_xlog()"); + +execute("mkdir -p $strTestPath/$strArchiveDir/0000000100000000"); + +# Test for archive log file 000000010000000000000001 +wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000001\$", 5); + +# Turn on log checksum for the next test +$dbh->disconnect(); +pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common"); +$strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 0, 1); +pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand); +$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser, + 'password', {AutoCommit => 1}); + +# Write another value into the test table +pg_execute($dbh, "insert into test values (2)"); +pg_execute($dbh, "select pg_switch_xlog()"); + +# Test for archive log file 000000010000000000000002 +wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5); + +# Turn on log compression and checksum for the next test +$dbh->disconnect(); +pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common"); +$strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 1, 1); +pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand); +$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser, + 'password', {AutoCommit => 1}); + +# Write another value into the test table +pg_execute($dbh, "insert into test values (3)"); +pg_execute($dbh, "select pg_switch_xlog()"); + +# Test for archive log file 000000010000000000000003 +wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5); + +$dbh->disconnect(); + +################################################################################ +# Stop the server +################################################################################ +#pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common"); + +################################################################################ +# Start an offline backup +################################################################################ +#pgbr_backup($strBackRestBinPath, "db");