#################################################################################################################################### # MANIFEST MODULE #################################################################################################################################### package BackRest::Manifest; use strict; use warnings FATAL => qw(all); use Carp qw(confess); use File::Basename qw(dirname basename); use Time::Local qw(timelocal); use Digest::SHA; use lib dirname($0); use BackRest::Exception qw(ERROR_CHECKSUM ERROR_FORMAT); use BackRest::Utility; use BackRest::File; # Exports use Exporter qw(import); our @EXPORT = qw(MANIFEST_PATH MANIFEST_FILE MANIFEST_LINK MANIFEST_SECTION_BACKUP MANIFEST_SECTION_BACKUP_OPTION MANIFEST_SECTION_BACKUP_PATH MANIFEST_SECTION_BACKUP_TABLESPACE MANIFEST_KEY_ARCHIVE_START MANIFEST_KEY_ARCHIVE_STOP MANIFEST_KEY_BASE MANIFEST_KEY_CHECKSUM MANIFEST_KEY_COMPRESS MANIFEST_KEY_HARDLINK MANIFEST_KEY_LABEL MANIFEST_KEY_PRIOR MANIFEST_KEY_REFERENCE MANIFEST_KEY_TIMESTAMP_COPY_START MANIFEST_KEY_TIMESTAMP_START MANIFEST_KEY_TIMESTAMP_STOP MANIFEST_KEY_TYPE MANIFEST_KEY_VERSION MANIFEST_SUBKEY_CHECKSUM MANIFEST_SUBKEY_DESTINATION MANIFEST_SUBKEY_FUTURE MANIFEST_SUBKEY_GROUP MANIFEST_SUBKEY_LINK MANIFEST_SUBKEY_MODE MANIFEST_SUBKEY_MODIFICATION_TIME MANIFEST_SUBKEY_PATH MANIFEST_SUBKEY_REFERENCE MANIFEST_SUBKEY_SIZE MANIFEST_SUBKEY_USER); #################################################################################################################################### # File/path constants #################################################################################################################################### use constant FILE_MANIFEST => 'backup.manifest'; push @EXPORT, qw(FILE_MANIFEST); #################################################################################################################################### # MANIFEST Constants #################################################################################################################################### use constant { MANIFEST_PATH => 'path', MANIFEST_FILE => 'file', MANIFEST_LINK => 'link', MANIFEST_SECTION_BACKUP => 'backup', MANIFEST_SECTION_BACKUP_OPTION => 'backup:option', MANIFEST_SECTION_BACKUP_PATH => 'backup:path', MANIFEST_SECTION_BACKUP_TABLESPACE => 'backup:tablespace', MANIFEST_KEY_ARCHIVE_START => 'archive-start', MANIFEST_KEY_ARCHIVE_STOP => 'archive-stop', MANIFEST_KEY_BASE => 'base', MANIFEST_KEY_CHECKSUM => 'checksum', MANIFEST_KEY_COMPRESS => 'compress', MANIFEST_KEY_FORMAT => 'format', MANIFEST_KEY_HARDLINK => 'hardlink', MANIFEST_KEY_LABEL => 'label', MANIFEST_KEY_PRIOR => 'prior', MANIFEST_KEY_REFERENCE => 'reference', MANIFEST_KEY_TIMESTAMP_COPY_START => 'timestamp-copy-start', MANIFEST_KEY_TIMESTAMP_START => 'timestamp-start', MANIFEST_KEY_TIMESTAMP_STOP => 'timestamp-stop', MANIFEST_KEY_TYPE => 'type', MANIFEST_KEY_VERSION => 'version', MANIFEST_SUBKEY_CHECKSUM => 'checksum', MANIFEST_SUBKEY_DESTINATION => 'link_destination', MANIFEST_SUBKEY_FUTURE => 'future', MANIFEST_SUBKEY_GROUP => 'group', MANIFEST_SUBKEY_LINK => 'link', MANIFEST_SUBKEY_MODE => 'mode', MANIFEST_SUBKEY_MODIFICATION_TIME => 'modification_time', MANIFEST_SUBKEY_PATH => 'path', MANIFEST_SUBKEY_REFERENCE => 'reference', MANIFEST_SUBKEY_SIZE => 'size', MANIFEST_SUBKEY_USER => 'user' }; #################################################################################################################################### # CONSTRUCTOR #################################################################################################################################### sub new { my $class = shift; # Class name my $strFileName = shift; # Manifest filename my $bLoad = shift; # Load the manifest? # Create the class hash my $self = {}; bless $self, $class; # Filename must be specified if (!defined($strFileName)) { confess &log(ASSERT, 'filename must be provided'); } # Set variables my $oManifest = {}; $self->{oManifest} = $oManifest; $self->{strFileName} = $strFileName; # Load the manifest if specified if (!(defined($bLoad) && $bLoad == false)) { ini_load($strFileName, $oManifest); # Make sure the manifest is valid by testing checksum my $strChecksum = $self->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_CHECKSUM); my $strTestChecksum = $self->hash(); if ($strChecksum ne $strTestChecksum) { confess &log(ERROR, "backup.manifest checksum is invalid, should be ${strTestChecksum}", ERROR_CHECKSUM); } # Make sure that the format is current, otherwise error my $iFormat = $self->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_FORMAT, undef, false, 0); if ($iFormat != FORMAT) { confess &log(ERROR, "backup format of ${strFileName} is ${iFormat} but " . FORMAT . ' is required by this version of ' . 'PgBackRest. If you are attempting an incr/diff backup you will need to take a new full backup. ' . "If you are trying to restore, you''ll need to use a version that supports format ${iFormat}." , ERROR_FORMAT); } } else { $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_FORMAT, undef, FORMAT); } return $self; } #################################################################################################################################### # SAVE # # Save the manifest. #################################################################################################################################### sub save { my $self = shift; # Create the checksum $self->hash(); # Save the config file ini_save($self->{strFileName}, $self->{oManifest}); } #################################################################################################################################### # HASH # # Generate hash for the manifest. #################################################################################################################################### sub hash { my $self = shift; my $oManifest = $self->{oManifest}; # Remove the old checksum $self->remove(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_CHECKSUM); my $oSHA = Digest::SHA->new('sha1'); # Calculate the checksum from section values foreach my $strSection ($self->keys()) { $oSHA->add($strSection); # Calculate the checksum from key values foreach my $strKey ($self->keys($strSection)) { $oSHA->add($strKey); my $strValue = $self->get($strSection, $strKey); if (!defined($strValue)) { confess &log(ASSERT, "section ${strSection}, key ${$strKey} has undef value"); } # Calculate the checksum from subkey values if (ref($strValue) eq "HASH") { foreach my $strSubKey ($self->keys($strSection, $strKey)) { my $strSubValue = $self->get($strSection, $strKey, $strSubKey); if (!defined($strSubValue)) { confess &log(ASSERT, "section ${strSection}, key ${strKey}, subkey ${strSubKey} has undef value"); } $oSHA->add($strSubValue); } } else { $oSHA->add($strValue); } } } # Set the new checksum my $strHash = $oSHA->hexdigest(); $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_CHECKSUM, undef, $strHash); return $strHash; } #################################################################################################################################### # GET # # Get a value. #################################################################################################################################### sub get { my $self = shift; my $strSection = shift; my $strValue = shift; my $strSubValue = shift; my $bRequired = shift; my $oDefault = shift; my $oManifest = $self->{oManifest}; # Section must always be defined if (!defined($strSection)) { confess &log(ASSERT, 'section is not defined'); } # Set default for required $bRequired = defined($bRequired) ? $bRequired : true; # Store the result my $oResult = undef; if (defined($strSubValue)) { if (!defined($strValue)) { confess &log(ASSERT, 'subvalue requested bu value is not defined'); } if (defined(${$oManifest}{$strSection}{$strValue})) { $oResult = ${$oManifest}{$strSection}{$strValue}{$strSubValue}; } } elsif (defined($strValue)) { if (defined(${$oManifest}{$strSection})) { $oResult = ${$oManifest}{$strSection}{$strValue}; } } else { $oResult = ${$oManifest}{$strSection}; } if (!defined($oResult) && $bRequired) { confess &log(ASSERT, "manifest section '$strSection'" . (defined($strValue) ? ", value '$strValue'" : '') . (defined($strSubValue) ? ", subvalue '$strSubValue'" : '') . ' is required but not defined'); } if (!defined($oResult) && defined($oDefault)) { $oResult = $oDefault; } return $oResult } #################################################################################################################################### # SET # # Set a value. #################################################################################################################################### sub set { my $self = shift; my $strSection = shift; my $strKey = shift; my $strSubKey = shift; my $strValue = shift; my $oManifest = $self->{oManifest}; # Make sure the keys are valid $self->valid($strSection, $strKey, $strSubKey); if (defined($strSubKey)) { ${$oManifest}{$strSection}{$strKey}{$strSubKey} = $strValue; } else { ${$oManifest}{$strSection}{$strKey} = $strValue; } } #################################################################################################################################### # REMOVE # # Remove a value. #################################################################################################################################### sub remove { my $self = shift; my $strSection = shift; my $strKey = shift; my $strSubKey = shift; my $strValue = shift; my $oManifest = $self->{oManifest}; # Make sure the keys are valid $self->valid($strSection, $strKey, $strSubKey, undef, true); if (defined($strSubKey)) { delete(${$oManifest}{$strSection}{$strKey}{$strSubKey}); } else { delete(${$oManifest}{$strSection}{$strKey}); } } #################################################################################################################################### # VALID # # Determine if section, key, subkey combination is valid. #################################################################################################################################### sub valid { my $self = shift; my $strSection = shift; my $strKey = shift; my $strSubKey = shift; my $strValue = shift; my $bDelete = shift; # Section and key must always be defined if (!defined($strSection) || !defined($strKey)) { confess &log(ASSERT, 'section or key is not defined'); } # Default bDelete $bDelete = defined($bDelete) ? $bDelete : false; if ($strSection =~ /^.*\:(file|path|link)$/ && $strSection !~ /^backup\:path$/) { if (!defined($strSubKey) && $bDelete) { return true; } my $strPath = (split(':', $strSection))[0]; my $strType = (split(':', $strSection))[1]; if ($strPath eq 'tablespace') { $strPath = (split(':', $strSection))[1]; $strType = (split(':', $strSection))[2]; } if (($strType eq 'path' || $strType eq 'file' || $strType eq 'link') && ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP)) { return true; } elsif (($strType eq 'path' || $strType eq 'file') && ($strSubKey eq MANIFEST_SUBKEY_MODE)) { return true; } elsif ($strType eq 'file' && ($strSubKey eq MANIFEST_SUBKEY_CHECKSUM || $strSubKey eq MANIFEST_SUBKEY_FUTURE || $strSubKey eq MANIFEST_SUBKEY_MODIFICATION_TIME || $strSubKey eq MANIFEST_SUBKEY_REFERENCE || $strSubKey eq MANIFEST_SUBKEY_SIZE)) { return true; } elsif ($strType eq 'link' && $strSubKey eq MANIFEST_SUBKEY_DESTINATION) { return true; } } if ($strSection eq MANIFEST_SECTION_BACKUP) { if ($strKey eq MANIFEST_KEY_ARCHIVE_START || $strKey eq MANIFEST_KEY_ARCHIVE_STOP || $strKey eq MANIFEST_KEY_CHECKSUM || $strKey eq MANIFEST_KEY_FORMAT || $strKey eq MANIFEST_KEY_LABEL || $strKey eq MANIFEST_KEY_PRIOR || $strKey eq MANIFEST_KEY_REFERENCE || $strKey eq MANIFEST_KEY_TIMESTAMP_COPY_START || $strKey eq MANIFEST_KEY_TIMESTAMP_START || $strKey eq MANIFEST_KEY_TIMESTAMP_STOP || $strKey eq MANIFEST_KEY_TYPE || $strKey eq MANIFEST_KEY_VERSION) { return true; } } elsif ($strSection eq MANIFEST_SECTION_BACKUP_OPTION) { if ($strKey eq MANIFEST_KEY_CHECKSUM || $strKey eq MANIFEST_KEY_COMPRESS || $strKey eq MANIFEST_KEY_HARDLINK) { return true; } } elsif ($strSection eq MANIFEST_SECTION_BACKUP_TABLESPACE) { if ($strSubKey eq 'link' || $strSubKey eq 'path') { return true; } } elsif ($strSection eq MANIFEST_SECTION_BACKUP_PATH) { if ($strKey eq 'base' || $strKey =~ /^tablespace\:.*$/) { return true; } } confess &log(ASSERT, "manifest section '${strSection}', key '${strKey}'" . (defined($strSubKey) ? ", subkey '$strSubKey'" : '') . ' is not valid'); } #################################################################################################################################### # epoch # # Retrieves a value in the format YYYY-MM-DD HH24:MI:SS and converts to epoch time. #################################################################################################################################### sub epoch { my $self = shift; my $strSection = shift; my $strKey = shift; my $strSubKey = shift; my $strValue = $self->get($strSection, $strKey, $strSubKey); my ($iYear, $iMonth, $iDay, $iHour, $iMinute, $iSecond) = split(/[\s\-\:]+/, $strValue); return timelocal($iSecond, $iMinute, $iHour, $iDay , $iMonth - 1, $iYear); } #################################################################################################################################### # KEYS # # Get a list of keys. #################################################################################################################################### sub keys { my $self = shift; my $strSection = shift; my $strKey = shift; if (defined($strSection)) { if ($self->test($strSection, $strKey)) { return sort(keys $self->get($strSection, $strKey)); } return []; } return sort(keys $self->{oManifest}); } #################################################################################################################################### # TEST # # Test a value to see if it equals the supplied test value. If no test value is given, tests that it is defined. #################################################################################################################################### sub test { my $self = shift; my $strSection = shift; my $strValue = shift; my $strSubValue = shift; my $strTest = shift; my $strResult = $self->get($strSection, $strValue, $strSubValue, false); if (defined($strResult)) { if (defined($strTest)) { return $strResult eq $strTest ? true : false; } return true; } return false; } #################################################################################################################################### # BUILD # # Build the manifest object. #################################################################################################################################### sub build { my $self = shift; my $oFile = shift; my $strDbClusterPath = shift; my $oLastManifest = shift; my $bNoStartStop = shift; my $oTablespaceMapRef = shift; my $strLevel = shift; &log(DEBUG, 'Manifest->build'); # If no level is defined then it must be base if (!defined($strLevel)) { $strLevel = 'base'; if (defined($oLastManifest)) { $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef, $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); } # If bNoStartStop then build the tablespace map from pg_tblspc path if ($bNoStartStop) { $oTablespaceMapRef = {}; my %oTablespaceManifestHash; $oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath . '/pg_tblspc', \%oTablespaceManifestHash); foreach my $strName (sort(CORE::keys $oTablespaceManifestHash{name})) { if ($strName eq '.' or $strName eq '..') { next; } if ($oTablespaceManifestHash{name}{$strName}{type} ne 'l') { confess &log(ERROR, "pg_tblspc/${strName} is not a link"); } &log(DEBUG, "Found tablespace ${strName}"); ${$oTablespaceMapRef}{oid}{$strName}{name} = $strName; } } } # Get the manifest for this level my %oManifestHash; $oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash); $self->set(MANIFEST_SECTION_BACKUP_PATH, $strLevel, undef, $strDbClusterPath); # Loop though all paths/files/links in the manifest foreach my $strName (sort(CORE::keys $oManifestHash{name})) { # Skip certain files during backup if (($strName =~ /^pg\_xlog\/.*/ && !$bNoStartStop) || # pg_xlog/ - this will be reconstructed $strName =~ /^postmaster\.pid$/ || # postmaster.pid - to avoid confusing postgres when restoring $strName =~ /^recovery\.conf$/) # recovery.conf - doesn't make sense to backup this file { next; } my $cType = $oManifestHash{name}{"${strName}"}{type}; my $strLinkDestination = $oManifestHash{name}{"${strName}"}{link_destination}; my $strSection = "${strLevel}:path"; if ($cType eq 'f') { $strSection = "${strLevel}:file"; } elsif ($cType eq 'l') { $strSection = "${strLevel}:link"; } elsif ($cType ne 'd') { confess &log(ASSERT, "unrecognized file type $cType for file $strName"); } # User and group required for all types $self->set($strSection, $strName, MANIFEST_SUBKEY_USER, $oManifestHash{name}{"${strName}"}{user}); $self->set($strSection, $strName, MANIFEST_SUBKEY_GROUP, $oManifestHash{name}{"${strName}"}{group}); # Mode for required file and path type only if ($cType eq 'f' || $cType eq 'd') { $self->set($strSection, $strName, MANIFEST_SUBKEY_MODE, $oManifestHash{name}{"${strName}"}{mode}); } # Modification time and size required for file type only if ($cType eq 'f') { $self->set($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME, $oManifestHash{name}{"${strName}"}{modification_time} + 0); $self->set($strSection, $strName, MANIFEST_SUBKEY_SIZE, $oManifestHash{name}{"${strName}"}{size} + 0); } # Link destination required for link type only if ($cType eq 'l') { $self->set($strSection, $strName, MANIFEST_SUBKEY_DESTINATION, $oManifestHash{name}{"${strName}"}{link_destination}); # If this is a tablespace then follow the link if (index($strName, 'pg_tblspc/') == 0 && $strLevel eq 'base') { my $strTablespaceOid = basename($strName); my $strTablespaceName = ${$oTablespaceMapRef}{oid}{$strTablespaceOid}{name}; $self->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName, MANIFEST_SUBKEY_LINK, $strTablespaceOid); $self->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName, MANIFEST_SUBKEY_PATH, $strLinkDestination); $self->build($oFile, $strLinkDestination, $oLastManifest, $bNoStartStop, $oTablespaceMapRef, "tablespace:${strTablespaceName}"); } } } # If this is the base level then do post-processing if ($strLevel eq 'base') { my $bTimeInFuture = false; my $lTimeBegin = $oFile->wait(PATH_DB_ABSOLUTE); # Loop through all backup paths (base and tablespaces) foreach my $strPathKey ($self->keys(MANIFEST_SECTION_BACKUP_PATH)) { my $strSection = "${strPathKey}:file"; # Make sure file section exists if ($self->test($strSection)) { # Loop though all files foreach my $strName ($self->keys($strSection)) { # If modification time is in the future (in this backup OR the last backup) set warning flag and do not # allow a reference if ($self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin || (defined($oLastManifest) && $oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))) { $bTimeInFuture = true; # Only mark as future if still in the future in the current backup if ($self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin) { $self->set($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y'); } } # Else check if modification time and size are unchanged since last backup elsif (defined($oLastManifest) && $oLastManifest->test($strSection, $strName) && $self->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) == $oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) && $self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) == $oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME)) { # Copy reference from previous backup if possible if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_REFERENCE)) { $self->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE, $oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE)); } # Otherwise the reference is to the previous backup else { $self->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE, $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL)); } # Copy the checksum from previous manifest if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM)) { $self->set($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM, $oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM)); } # Build the manifest reference list - not used for processing but is useful for debugging my $strFileReference = $self->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE); my $strManifestReference = $self->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef, false); if (!defined($strManifestReference)) { $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef, $strFileReference); } else { if ($strManifestReference !~ /^$strFileReference|,$strFileReference/) { $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef, $strManifestReference . ",${strFileReference}"); } } } } } } # Warn if any files in the current backup are in the future if ($bTimeInFuture) { &log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions"); } # Record the time when copying will start $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, timestamp_string_get(undef, $lTimeBegin + 1)); } } 1;