diff --git a/bin/pg_backrest.pl b/bin/pg_backrest.pl index 0ccef4fe4..2336dd208 100755 --- a/bin/pg_backrest.pl +++ b/bin/pg_backrest.pl @@ -17,6 +17,7 @@ use Pod::Usage; use lib dirname($0) . '/../lib'; use BackRest::Utility; +use BackRest::Config; use BackRest::File; use BackRest::Backup; use BackRest::Db; @@ -63,45 +64,6 @@ use constant OP_EXPIRE => 'expire' }; -#################################################################################################################################### -# Configuration constants - configuration sections and keys -#################################################################################################################################### -use constant -{ - CONFIG_SECTION_COMMAND => 'command', - CONFIG_SECTION_COMMAND_OPTION => 'command:option', - CONFIG_SECTION_LOG => 'log', - CONFIG_SECTION_BACKUP => 'backup', - CONFIG_SECTION_ARCHIVE => 'archive', - CONFIG_SECTION_RETENTION => 'retention', - CONFIG_SECTION_STANZA => 'stanza', - - CONFIG_KEY_USER => 'user', - CONFIG_KEY_HOST => 'host', - CONFIG_KEY_PATH => 'path', - - CONFIG_KEY_THREAD_MAX => 'thread-max', - CONFIG_KEY_THREAD_TIMEOUT => 'thread-timeout', - CONFIG_KEY_HARDLINK => 'hardlink', - CONFIG_KEY_ARCHIVE_REQUIRED => 'archive-required', - CONFIG_KEY_ARCHIVE_MAX_MB => 'archive-max-mb', - CONFIG_KEY_START_FAST => 'start-fast', - CONFIG_KEY_COMPRESS_ASYNC => 'compress-async', - - CONFIG_KEY_LEVEL_FILE => 'level-file', - CONFIG_KEY_LEVEL_CONSOLE => 'level-console', - - CONFIG_KEY_COMPRESS => 'compress', - CONFIG_KEY_CHECKSUM => 'checksum', - CONFIG_KEY_PSQL => 'psql', - CONFIG_KEY_REMOTE => 'remote', - - CONFIG_KEY_FULL_RETENTION => 'full-retention', - CONFIG_KEY_DIFFERENTIAL_RETENTION => 'differential-retention', - CONFIG_KEY_ARCHIVE_RETENTION_TYPE => 'archive-retention-type', - CONFIG_KEY_ARCHIVE_RETENTION => 'archive-retention' -}; - #################################################################################################################################### # Command line parameters #################################################################################################################################### @@ -153,72 +115,15 @@ if ($bHelp) # Set test parameters test_set($bTest, $iTestDelay); +# Load the config file +config_load($strConfigFile, $strStanza); + #################################################################################################################################### # Global variables #################################################################################################################################### -my %oConfig; # Configuration hash my $oRemote; # Remote object my $strRemote; # Defines which side is remote, DB or BACKUP -#################################################################################################################################### -# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) -#################################################################################################################################### -sub config_key_load -{ - my $strSection = shift; - my $strKey = shift; - my $bRequired = shift; - my $strDefault = shift; - - # Default is that the key is not required - if (!defined($bRequired)) - { - $bRequired = false; - } - - my $strValue; - - # Look in the default stanza section - if ($strSection eq CONFIG_SECTION_STANZA) - { - $strValue = $oConfig{"${strStanza}"}{"${strKey}"}; - } - # Else look in the supplied section - else - { - # First check the stanza section - $strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"}; - - # If the stanza section value is undefined then check global - if (!defined($strValue)) - { - $strValue = $oConfig{"global:${strSection}"}{"${strKey}"}; - } - } - - if (!defined($strValue) && $bRequired) - { - if (defined($strDefault)) - { - return $strDefault; - } - - confess &log(ERROR, 'config value ' . (defined($strSection) ? $strSection : '[stanza]') . "->${strKey} is undefined"); - } - - if ($strSection eq CONFIG_SECTION_COMMAND) - { - my $strOption = config_key_load(CONFIG_SECTION_COMMAND_OPTION, $strKey); - - if (defined($strOption)) - { - $strValue =~ s/\%option\%/${strOption}/g; - } - } - - return $strValue; -} - #################################################################################################################################### # REMOTE_EXIT - Close the remote object if it exists #################################################################################################################################### @@ -302,26 +207,6 @@ if (defined($strType) && $strOperation ne OP_BACKUP) confess &log(ERROR, 'type can only be specified for the backup operation') } -#################################################################################################################################### -# LOAD CONFIG FILE -#################################################################################################################################### -if (!defined($strConfigFile)) -{ - $strConfigFile = '/etc/pg_backrest.conf'; -} - -config_load($strConfigFile, \%oConfig); - -# Load and check the cluster -if (!defined($strStanza)) -{ - confess 'a backup stanza must be specified'; -} - -# Set the log levels -log_level_set(uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, INFO)), - uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, ERROR))); - #################################################################################################################################### # DETERMINE IF THERE IS A REMOTE #################################################################################################################################### diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index 0a4fd7e6a..7ec4dbf73 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -1361,7 +1361,7 @@ sub backup if (defined($strBackupLastPath)) { - config_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/${strBackupLastPath}/backup.manifest", \%oLastManifest); + ini_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/${strBackupLastPath}/backup.manifest", \%oLastManifest); if (!defined($oLastManifest{backup}{label})) { @@ -1471,7 +1471,7 @@ sub backup { # Load the aborted manifest my %oAbortedManifest; - config_load("${strBackupTmpPath}/backup.manifest", \%oAbortedManifest); + ini_load("${strBackupTmpPath}/backup.manifest", \%oAbortedManifest); # Default values if they are not set my $strAbortedType = defined($oAbortedManifest{backup}{type}) ? @@ -1529,7 +1529,7 @@ sub backup close($hVersionFile); # Save the backup conf file with the manifest - config_save($strBackupConfFile, \%oBackupManifest); + ini_save($strBackupConfFile, \%oBackupManifest); # Perform the backup backup_file($strDbClusterPath, \%oBackupManifest); @@ -1550,7 +1550,7 @@ sub backup if ($bArchiveRequired) { # Save the backup conf file second time - before getting archive logs in case that fails - config_save($strBackupConfFile, \%oBackupManifest); + ini_save($strBackupConfFile, \%oBackupManifest); # After the backup has been stopped, need to make a copy of the archive logs need to make the db consistent &log(DEBUG, "retrieving archive logs ${strArchiveStart}:${strArchiveStop}"); @@ -1608,7 +1608,7 @@ sub backup ${oBackupManifest}{backup}{label} = $strBackupPath; # Save the backup conf file final time - config_save($strBackupConfFile, \%oBackupManifest); + ini_save($strBackupConfFile, \%oBackupManifest); &log(INFO, "new backup label: ${strBackupPath}"); @@ -1836,7 +1836,7 @@ sub backup_expire &log(INFO, 'archive retention based on backup ' . $strArchiveRetentionBackup); my %oManifest; - config_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/${strArchiveRetentionBackup}/backup.manifest", \%oManifest); + ini_load($oFile->path_get(PATH_BACKUP_CLUSTER) . "/${strArchiveRetentionBackup}/backup.manifest", \%oManifest); my $strArchiveLast = ${oManifest}{backup}{'archive-start'}; if (!defined($strArchiveLast)) diff --git a/lib/BackRest/Config.pm b/lib/BackRest/Config.pm new file mode 100644 index 000000000..2bee7d25e --- /dev/null +++ b/lib/BackRest/Config.pm @@ -0,0 +1,171 @@ +#################################################################################################################################### +# CONFIG MODULE +#################################################################################################################################### +package BackRest::Config; + +use threads; +use strict; +use warnings; +use Carp; + +use File::Basename; + +use lib dirname($0) . '/../lib'; +use BackRest::Utility; + +use Exporter qw(import); + +our @EXPORT = qw(config_load config_key_load + + CONFIG_SECTION_COMMAND CONFIG_SECTION_COMMAND_OPTION CONFIG_SECTION_LOG CONFIG_SECTION_BACKUP + CONFIG_SECTION_ARCHIVE CONFIG_SECTION_RETENTION CONFIG_SECTION_STANZA + + CONFIG_KEY_USER CONFIG_KEY_HOST CONFIG_KEY_PATH + + CONFIG_KEY_THREAD_MAX CONFIG_KEY_THREAD_TIMEOUT CONFIG_KEY_HARDLINK CONFIG_KEY_ARCHIVE_REQUIRED + CONFIG_KEY_ARCHIVE_MAX_MB CONFIG_KEY_START_FAST CONFIG_KEY_COMPRESS_ASYNC + + CONFIG_KEY_LEVEL_FILE CONFIG_KEY_LEVEL_CONSOLE + + CONFIG_KEY_COMPRESS CONFIG_KEY_CHECKSUM CONFIG_KEY_PSQL CONFIG_KEY_REMOTE + + CONFIG_KEY_FULL_RETENTION CONFIG_KEY_DIFFERENTIAL_RETENTION CONFIG_KEY_ARCHIVE_RETENTION_TYPE + CONFIG_KEY_ARCHIVE_RETENTION); + +#################################################################################################################################### +# Configuration constants - configuration sections and keys +#################################################################################################################################### +use constant +{ + CONFIG_SECTION_COMMAND => 'command', + CONFIG_SECTION_COMMAND_OPTION => 'command:option', + CONFIG_SECTION_LOG => 'log', + CONFIG_SECTION_BACKUP => 'backup', + CONFIG_SECTION_ARCHIVE => 'archive', + CONFIG_SECTION_RETENTION => 'retention', + CONFIG_SECTION_STANZA => 'stanza', + + CONFIG_KEY_USER => 'user', + CONFIG_KEY_HOST => 'host', + CONFIG_KEY_PATH => 'path', + + CONFIG_KEY_THREAD_MAX => 'thread-max', + CONFIG_KEY_THREAD_TIMEOUT => 'thread-timeout', + CONFIG_KEY_HARDLINK => 'hardlink', + CONFIG_KEY_ARCHIVE_REQUIRED => 'archive-required', + CONFIG_KEY_ARCHIVE_MAX_MB => 'archive-max-mb', + CONFIG_KEY_START_FAST => 'start-fast', + CONFIG_KEY_COMPRESS_ASYNC => 'compress-async', + + CONFIG_KEY_LEVEL_FILE => 'level-file', + CONFIG_KEY_LEVEL_CONSOLE => 'level-console', + + CONFIG_KEY_COMPRESS => 'compress', + CONFIG_KEY_CHECKSUM => 'checksum', + CONFIG_KEY_PSQL => 'psql', + CONFIG_KEY_REMOTE => 'remote', + + CONFIG_KEY_FULL_RETENTION => 'full-retention', + CONFIG_KEY_DIFFERENTIAL_RETENTION => 'differential-retention', + CONFIG_KEY_ARCHIVE_RETENTION_TYPE => 'archive-retention-type', + CONFIG_KEY_ARCHIVE_RETENTION => 'archive-retention' +}; + +#################################################################################################################################### +# Global variables +#################################################################################################################################### +my %oConfig; # Configuration hash + +my $strStanza; # Config stanza + +#################################################################################################################################### +# CONFIG_LOAD +# +# Load config file. +#################################################################################################################################### +sub config_load +{ + my $strFile = shift; # Full path to ini file to load from + my $strStanzaParam = shift; # Stanza specified on command line + + if (!defined($strFile)) + { + $strFile = '/etc/pg_backrest.conf'; + } + + ini_load($strFile, \%oConfig); + + # Load and check the cluster + if (!defined($strStanzaParam)) + { + confess 'a backup stanza must be specified'; + } + + # Set the log levels + log_level_set(uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, INFO)), + uc(config_key_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, ERROR))); + + # Set globals + $strStanza = $strStanzaParam; +} + +#################################################################################################################################### +# CONFIG_KEY_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false) +#################################################################################################################################### +sub config_key_load +{ + my $strSection = shift; + my $strKey = shift; + my $bRequired = shift; + my $strDefault = shift; + + # Default is that the key is not required + if (!defined($bRequired)) + { + $bRequired = false; + } + + my $strValue; + + # Look in the default stanza section + if ($strSection eq CONFIG_SECTION_STANZA) + { + $strValue = $oConfig{"${strStanza}"}{"${strKey}"}; + } + # Else look in the supplied section + else + { + # First check the stanza section + $strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"}; + + # If the stanza section value is undefined then check global + if (!defined($strValue)) + { + $strValue = $oConfig{"global:${strSection}"}{"${strKey}"}; + } + } + + if (!defined($strValue) && $bRequired) + { + if (defined($strDefault)) + { + return $strDefault; + } + + confess &log(ERROR, 'config value ' . (defined($strSection) ? $strSection : '[stanza]') . "->${strKey} is undefined"); + } + + if ($strSection eq CONFIG_SECTION_COMMAND) + { + my $strOption = config_key_load(CONFIG_SECTION_COMMAND_OPTION, $strKey); + + if (defined($strOption)) + { + $strValue =~ s/\%option\%/${strOption}/g; + } + } + + return $strValue; +} + +1; diff --git a/lib/BackRest/Utility.pm b/lib/BackRest/Utility.pm index 14b5b8f4d..f893f6326 100644 --- a/lib/BackRest/Utility.pm +++ b/lib/BackRest/Utility.pm @@ -22,7 +22,7 @@ our @EXPORT = qw(version_get data_hash_build trim common_prefix wait_for_file file_size_format execute log log_file_set log_level_set test_set test_check lock_file_create lock_file_remove - config_save config_load timestamp_string_get timestamp_file_string_get + ini_save ini_load timestamp_string_get timestamp_file_string_get TRACE DEBUG ERROR ASSERT WARN INFO OFF true false TEST TEST_ENCLOSE TEST_MANIFEST_BUILD); @@ -506,16 +506,16 @@ sub log } #################################################################################################################################### -# CONFIG_LOAD +# INI_LOAD # -# Load configuration file from standard INI format to a hash. +# Load file from standard INI format to a hash. #################################################################################################################################### -sub config_load +sub ini_load { - my $strFile = shift; # Full path to config file to load from - my $oConfig = shift; # Reference to the hash where config data will be stored + my $strFile = shift; # Full path to ini file to load from + my $oConfig = shift; # Reference to the hash where ini data will be stored - # Open the config file for reading + # Open the ini file for reading my $hFile; my $strSection; @@ -565,16 +565,16 @@ sub config_load } #################################################################################################################################### -# CONFIG_SAVE +# INI_SAVE # -# Save configuration file from a hash to standard INI format. +# Save from a hash to standard INI format. #################################################################################################################################### -sub config_save +sub ini_save { - my $strFile = shift; # Full path to config file to save to - my $oConfig = shift; # Reference to the hash where config data is stored + my $strFile = shift; # Full path to ini file to save to + my $oConfig = shift; # Reference to the hash where ini data is stored - # Open the config file for writing + # Open the ini file for writing my $hFile; my $bFirst = true; diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 47b39bab5..20d44b73f 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -506,15 +506,15 @@ sub BackRestTestBackup_CompareBackup } my %oActualManifest; - config_load($oFile->path_get(PATH_BACKUP_CLUSTER, $strBackup) . '/backup.manifest', \%oActualManifest); + ini_load($oFile->path_get(PATH_BACKUP_CLUSTER, $strBackup) . '/backup.manifest', \%oActualManifest); ${$oExpectedManifestRef}{backup}{'timestamp-start'} = $oActualManifest{backup}{'timestamp-start'}; ${$oExpectedManifestRef}{backup}{'timestamp-stop'} = $oActualManifest{backup}{'timestamp-stop'}; my $strTestPath = BackRestTestCommon_TestPathGet(); - config_save("${strTestPath}/actual.manifest", \%oActualManifest); - config_save("${strTestPath}/expected.manifest", $oExpectedManifestRef); + ini_save("${strTestPath}/actual.manifest", \%oActualManifest); + ini_save("${strTestPath}/expected.manifest", $oExpectedManifestRef); BackRestTestCommon_Execute("diff ${strTestPath}/expected.manifest ${strTestPath}/actual.manifest"); diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index b63ff2185..0224e1325 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -441,7 +441,7 @@ sub BackRestTestCommon_ConfigCreate # Write out the configuration file my $strFile = BackRestTestCommon_TestPathGet() . '/pg_backrest.conf'; - config_save($strFile, \%oParamHash); + ini_save($strFile, \%oParamHash); # Move the configuration file based on local if ($strLocal eq 'db') diff --git a/test/lib/BackRestTest/UtilityTest.pm b/test/lib/BackRestTest/UtilityTest.pm index 5aff36ebf..e84112c16 100755 --- a/test/lib/BackRestTest/UtilityTest.pm +++ b/test/lib/BackRestTest/UtilityTest.pm @@ -96,18 +96,18 @@ sub BackRestTestUtility_Test # Save the test config my $strFile = "${strTestPath}/config.cfg"; - config_save($strFile, \%oConfig); + ini_save($strFile, \%oConfig); my $strConfigHash = $oFile->hash(PATH_ABSOLUTE, $strFile); # Reload the test config my %oConfigTest; - config_load($strFile, \%oConfigTest); + ini_load($strFile, \%oConfigTest); # Resave the test config and compare hashes my $strFileTest = "${strTestPath}/config-test.cfg"; - config_save($strFileTest, \%oConfigTest); + ini_save($strFileTest, \%oConfigTest); my $strConfigTestHash = $oFile->hash(PATH_ABSOLUTE, $strFileTest);