db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-09-16 09:06:18 +02:00
Code Issues Releases Activity

Moved manifest build to manifest object.

Browse Source
This commit is contained in:
David Steele
2015-01-31 18:15:08 -05:00
parent cd500484a2
commit d7b669d560
2 changed files with 191 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
lib/BackRest/Backup.pm
View File

@@ -825,192 +825,6 @@ sub backup_tmp_clean
}
}
####################################################################################################################################
# BACKUP_MANIFEST_BUILD - Create the backup manifest
####################################################################################################################################
sub backup_manifest_build
{
my $strDbClusterPath = shift;
my $oBackupManifest = shift;
my $oLastManifest = shift;
my $oTablespaceMapRef = shift;
my $strLevel = shift;
# If no level is defined then it must be base
if (!defined($strLevel))
{
$strLevel = 'base';
}
# Get the manifest for this level
my %oManifestHash;
$oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash);
$oBackupManifest->set(MANIFEST_SECTION_BACKUP_PATH, $strLevel, undef, $strDbClusterPath);
# Loop though all paths/files/links in the manifest
foreach my $strName (sort(keys $oManifestHash{name}))
{
# Skip certain files during backup
if (($strName =~ /^pg\_xlog\/.*/ && !$bNoStartStop) || # pg_xlog/ - this will be reconstructed
$strName =~ /^postmaster\.pid$/ || # postmaster.pid - to avoid confusing postgres when restoring
$strName =~ /^recovery\.conf$/) # recovery.conf - doesn't make sense to backup this file
{
next;
}
my $cType = $oManifestHash{name}{"${strName}"}{type};
my $strLinkDestination = $oManifestHash{name}{"${strName}"}{link_destination};
my $strSection = "${strLevel}:path";
if ($cType eq 'f')
{
$strSection = "${strLevel}:file";
}
elsif ($cType eq 'l')
{
$strSection = "${strLevel}:link";
}
elsif ($cType ne 'd')
{
confess &log(ASSERT, "unrecognized file type $cType for file $strName");
}
# User and group required for all types
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_USER, $oManifestHash{name}{"${strName}"}{user});
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_GROUP, $oManifestHash{name}{"${strName}"}{group});
# Mode for required file and path type only
if ($cType eq 'f' || $cType eq 'd')
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_MODE, $oManifestHash{name}{"${strName}"}{permission});
}
# Modification time and size required for file type only
if ($cType eq 'f')
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME,
$oManifestHash{name}{"${strName}"}{modification_time} + 0);
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_SIZE, $oManifestHash{name}{"${strName}"}{size} + 0);
}
# Link destination required for link type only
if ($cType eq 'l')
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_DESTINATION,
$oManifestHash{name}{"${strName}"}{link_destination});
# If this is a tablespace then follow the link
if (index($strName, 'pg_tblspc/') == 0 && $strLevel eq 'base')
{
my $strTablespaceOid = basename($strName);
my $strTablespaceName = ${$oTablespaceMapRef}{oid}{"${strTablespaceOid}"}{name};
$oBackupManifest->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName,
MANIFEST_SUBKEY_LINK, $strTablespaceOid);
$oBackupManifest->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName,
MANIFEST_SUBKEY_PATH, $strLinkDestination);
backup_manifest_build($strLinkDestination, $oBackupManifest, undef,
$oTablespaceMapRef, "tablespace:${strTablespaceName}");
}
}
}
# If this is the base level then do post-processing
if ($strLevel eq 'base')
{
my $bTimeInFuture = false;
my $lTimeBegin = $oFile->wait(PATH_DB_ABSOLUTE);
# Loop through all backup paths (base and tablespaces)
foreach my $strPathKey ($oBackupManifest->keys(MANIFEST_SECTION_BACKUP_PATH))
{
my $strSection = "${strPathKey}:file";
# Make sure file section exists
if ($oBackupManifest->test($strSection))
{
# Loop though all files
foreach my $strName ($oBackupManifest->keys($strSection))
{
# If modification time is in the future (in this backup OR the last backup) set warning flag and do not
# allow a reference
if ($oBackupManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin ||
(defined($oLastManifest) && $oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))
{
$bTimeInFuture = true;
# Only mark as future if still in the future in the current backup
if ($oBackupManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin)
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y');
}
}
# Else check if modification time and size are unchanged since last backup
elsif (defined($oLastManifest) && $oLastManifest->test($strSection, $strName) &&
$oBackupManifest->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) ==
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) &&
$oBackupManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) ==
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME))
{
# Copy reference from previous backup if possible
if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_REFERENCE))
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE,
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE));
}
# Otherwise the reference is to the previous backup
else
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE,
$oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
}
# Copy the checksum from previous manifest
if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM))
{
$oBackupManifest->set($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM,
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM));
}
# Build the manifest reference list - not used for processing but is useful for debugging
my $strFileReference = $oBackupManifest->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE);
my $strManifestReference = $oBackupManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE,
undef, false);
if (!defined($strManifestReference))
{
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef, $strFileReference);
}
else
{
if ($strManifestReference !~ /^$strFileReference|,$strFileReference/)
{
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef,
$strManifestReference . ",${strFileReference}");
}
}
}
}
}
}
# Warn if any files in the current backup are in the future
if ($bTimeInFuture)
{
&log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions");
}
# Record the time when copying will start
$oBackupManifest->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
timestamp_string_get(undef, $lTimeBegin + 1));
}
}
####################################################################################################################################
# BACKUP_FILE - Performs the file level backup
#
@@ -1521,7 +1335,7 @@ sub backup
$oDb->tablespace_map_get(\%oTablespaceMap);
}
backup_manifest_build($strDbClusterPath, $oBackupManifest, $oLastManifest, \%oTablespaceMap);
$oBackupManifest->build($oFile, $strDbClusterPath, $oLastManifest, $bNoStartStop, \%oTablespaceMap);
&log(TEST, TEST_MANIFEST_BUILD);
# Check if an aborted backup exists for this stanza

191
lib/BackRest/Manifest.pm
View File

@@ -8,12 +8,13 @@ use strict;
use warnings;
use Carp;
use File::Basename qw(dirname);
use File::Basename qw(dirname basename);
use Time::Local qw(timelocal);
use Digest::SHA;
use lib dirname($0);
use BackRest::Utility;
use BackRest::File;
# Exports
use Exporter qw(import);
@@ -501,4 +502,192 @@ sub test
return false;
}
####################################################################################################################################
# BUILD - Build the backup manifest
####################################################################################################################################
sub build
{
my $self = shift;
my $oFile = shift;
my $strDbClusterPath = shift;
my $oLastManifest = shift;
my $bNoStartStop = shift;
my $oTablespaceMapRef = shift;
my $strLevel = shift;
# If no level is defined then it must be base
if (!defined($strLevel))
{
$strLevel = 'base';
}
# Get the manifest for this level
my %oManifestHash;
$oFile->manifest(PATH_DB_ABSOLUTE, $strDbClusterPath, \%oManifestHash);
$self->set(MANIFEST_SECTION_BACKUP_PATH, $strLevel, undef, $strDbClusterPath);
# Loop though all paths/files/links in the manifest
foreach my $strName (sort(CORE::keys $oManifestHash{name}))
{
# Skip certain files during backup
if (($strName =~ /^pg\_xlog\/.*/ && !$bNoStartStop) || # pg_xlog/ - this will be reconstructed
$strName =~ /^postmaster\.pid$/ || # postmaster.pid - to avoid confusing postgres when restoring
$strName =~ /^recovery\.conf$/) # recovery.conf - doesn't make sense to backup this file
{
next;
}
my $cType = $oManifestHash{name}{"${strName}"}{type};
my $strLinkDestination = $oManifestHash{name}{"${strName}"}{link_destination};
my $strSection = "${strLevel}:path";
if ($cType eq 'f')
{
$strSection = "${strLevel}:file";
}
elsif ($cType eq 'l')
{
$strSection = "${strLevel}:link";
}
elsif ($cType ne 'd')
{
confess &log(ASSERT, "unrecognized file type $cType for file $strName");
}
# User and group required for all types
$self->set($strSection, $strName, MANIFEST_SUBKEY_USER, $oManifestHash{name}{"${strName}"}{user});
$self->set($strSection, $strName, MANIFEST_SUBKEY_GROUP, $oManifestHash{name}{"${strName}"}{group});
# Mode for required file and path type only
if ($cType eq 'f' || $cType eq 'd')
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_MODE, $oManifestHash{name}{"${strName}"}{permission});
}
# Modification time and size required for file type only
if ($cType eq 'f')
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME,
$oManifestHash{name}{"${strName}"}{modification_time} + 0);
$self->set($strSection, $strName, MANIFEST_SUBKEY_SIZE, $oManifestHash{name}{"${strName}"}{size} + 0);
}
# Link destination required for link type only
if ($cType eq 'l')
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_DESTINATION,
$oManifestHash{name}{"${strName}"}{link_destination});
# If this is a tablespace then follow the link
if (index($strName, 'pg_tblspc/') == 0 && $strLevel eq 'base')
{
my $strTablespaceOid = basename($strName);
my $strTablespaceName = ${$oTablespaceMapRef}{oid}{"${strTablespaceOid}"}{name};
$self->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName,
MANIFEST_SUBKEY_LINK, $strTablespaceOid);
$self->set(MANIFEST_SECTION_BACKUP_TABLESPACE, $strTablespaceName,
MANIFEST_SUBKEY_PATH, $strLinkDestination);
$self->build($oFile, $strLinkDestination, $oLastManifest, $bNoStartStop, $oTablespaceMapRef,
"tablespace:${strTablespaceName}");
}
}
}
# If this is the base level then do post-processing
if ($strLevel eq 'base')
{
my $bTimeInFuture = false;
my $lTimeBegin = $oFile->wait(PATH_DB_ABSOLUTE);
# Loop through all backup paths (base and tablespaces)
foreach my $strPathKey ($self->keys(MANIFEST_SECTION_BACKUP_PATH))
{
my $strSection = "${strPathKey}:file";
# Make sure file section exists
if ($self->test($strSection))
{
# Loop though all files
foreach my $strName ($self->keys($strSection))
{
# If modification time is in the future (in this backup OR the last backup) set warning flag and do not
# allow a reference
if ($self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin ||
(defined($oLastManifest) && $oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))
{
$bTimeInFuture = true;
# Only mark as future if still in the future in the current backup
if ($self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) > $lTimeBegin)
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_FUTURE, 'y');
}
}
# Else check if modification time and size are unchanged since last backup
elsif (defined($oLastManifest) && $oLastManifest->test($strSection, $strName) &&
$self->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) ==
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_SIZE) &&
$self->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME) ==
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_MODIFICATION_TIME))
{
# Copy reference from previous backup if possible
if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_REFERENCE))
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE,
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE));
}
# Otherwise the reference is to the previous backup
else
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_REFERENCE,
$oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
}
# Copy the checksum from previous manifest
if ($oLastManifest->test($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM))
{
$self->set($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM,
$oLastManifest->get($strSection, $strName, MANIFEST_SUBKEY_CHECKSUM));
}
# Build the manifest reference list - not used for processing but is useful for debugging
my $strFileReference = $self->get($strSection, $strName, MANIFEST_SUBKEY_REFERENCE);
my $strManifestReference = $self->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE,
undef, false);
if (!defined($strManifestReference))
{
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef, $strFileReference);
}
else
{
if ($strManifestReference !~ /^$strFileReference|,$strFileReference/)
{
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_REFERENCE, undef,
$strManifestReference . ",${strFileReference}");
}
}
}
}
}
}
# Warn if any files in the current backup are in the future
if ($bTimeInFuture)
{
&log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions");
}
# Record the time when copying will start
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef,
timestamp_string_get(undef, $lTimeBegin + 1));
}
}
1;