pgbackrest/test/lib/pgBackRestTest/Env/Manifest.pm

####################################################################################################################################
# MANIFEST MODULE
####################################################################################################################################
package pgBackRestTest::Env::Manifest;
use parent 'pgBackRestDoc::Common::Ini';

use strict;
use warnings FATAL => qw(all);
use Carp qw(confess);

use Exporter qw(import);
    our @EXPORT = qw();
use File::Basename qw(dirname basename);
use Time::Local qw(timelocal);

use pgBackRestDoc::Common::Exception;
use pgBackRestDoc::Common::Ini;
use pgBackRestDoc::Common::Log;

use pgBackRestTest::Common::DbVersion;
use pgBackRestTest::Common::StorageRepo;
use pgBackRestTest::Common::Wait;

####################################################################################################################################
# File/path constants
####################################################################################################################################
use constant PATH_BACKUP_HISTORY                                    => 'backup.history';
    push @EXPORT, qw(PATH_BACKUP_HISTORY);
use constant FILE_MANIFEST                                          => 'backup.manifest';
    push @EXPORT, qw(FILE_MANIFEST);
use constant FILE_MANIFEST_COPY                                     => FILE_MANIFEST . INI_COPY_EXT;
    push @EXPORT, qw(FILE_MANIFEST_COPY);

####################################################################################################################################
# Default match factor
####################################################################################################################################
use constant MANIFEST_DEFAULT_MATCH_FACTOR                          => 0.1;
    push @EXPORT, qw(MANIFEST_DEFAULT_MATCH_FACTOR);

####################################################################################################################################
# MANIFEST Constants
####################################################################################################################################
use constant MANIFEST_TARGET_PGDATA                                 => 'pg_data';
    push @EXPORT, qw(MANIFEST_TARGET_PGDATA);
use constant MANIFEST_TARGET_PGTBLSPC                               => 'pg_tblspc';
    push @EXPORT, qw(MANIFEST_TARGET_PGTBLSPC);

use constant MANIFEST_VALUE_PATH                                    => 'path';
    push @EXPORT, qw(MANIFEST_VALUE_PATH);
use constant MANIFEST_VALUE_LINK                                    => 'link';
    push @EXPORT, qw(MANIFEST_VALUE_LINK);

# Manifest sections
use constant MANIFEST_SECTION_BACKUP                                => 'backup';
    push @EXPORT, qw(MANIFEST_SECTION_BACKUP);
use constant MANIFEST_SECTION_BACKUP_DB                             => 'backup:db';
    push @EXPORT, qw(MANIFEST_SECTION_BACKUP_DB);
use constant MANIFEST_SECTION_BACKUP_INFO                           => 'backup:info';
    push @EXPORT, qw(MANIFEST_SECTION_BACKUP_INFO);
use constant MANIFEST_SECTION_BACKUP_OPTION                         => 'backup:option';
    push @EXPORT, qw(MANIFEST_SECTION_BACKUP_OPTION);
use constant MANIFEST_SECTION_BACKUP_TARGET                         => 'backup:target';
    push @EXPORT, qw(MANIFEST_SECTION_BACKUP_TARGET);
use constant MANIFEST_SECTION_DB                                    => 'db';
    push @EXPORT, qw(MANIFEST_SECTION_DB);
use constant MANIFEST_SECTION_TARGET_PATH                           => 'target:path';
    push @EXPORT, qw(MANIFEST_SECTION_TARGET_PATH);
use constant MANIFEST_SECTION_TARGET_FILE                           => 'target:file';
    push @EXPORT, qw(MANIFEST_SECTION_TARGET_FILE);
use constant MANIFEST_SECTION_TARGET_LINK                           => 'target:link';
    push @EXPORT, qw(MANIFEST_SECTION_TARGET_LINK);

# Backup metadata required for restores
use constant MANIFEST_KEY_ARCHIVE_START                             => 'backup-archive-start';
    push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_START);
use constant MANIFEST_KEY_ARCHIVE_STOP                              => 'backup-archive-stop';
    push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_STOP);
use constant MANIFEST_KEY_LABEL                                     => 'backup-label';
    push @EXPORT, qw(MANIFEST_KEY_LABEL);
use constant MANIFEST_KEY_LSN_START                                 => 'backup-lsn-start';
    push @EXPORT, qw(MANIFEST_KEY_LSN_START);
use constant MANIFEST_KEY_LSN_STOP                                  => 'backup-lsn-stop';
    push @EXPORT, qw(MANIFEST_KEY_LSN_STOP);
use constant MANIFEST_KEY_PRIOR                                     => 'backup-prior';
    push @EXPORT, qw(MANIFEST_KEY_PRIOR);
use constant MANIFEST_KEY_TIMESTAMP_COPY_START                      => 'backup-timestamp-copy-start';
    push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_COPY_START);
use constant MANIFEST_KEY_TIMESTAMP_START                           => 'backup-timestamp-start';
    push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_START);
use constant MANIFEST_KEY_TIMESTAMP_STOP                            => 'backup-timestamp-stop';
    push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_STOP);
use constant MANIFEST_KEY_TYPE                                      => 'backup-type';
    push @EXPORT, qw(MANIFEST_KEY_TYPE);

# Options that were set when the backup was made
use constant MANIFEST_KEY_BACKUP_STANDBY                            => 'option-backup-standby';
    push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY);
use constant MANIFEST_KEY_HARDLINK                                  => 'option-hardlink';
    push @EXPORT, qw(MANIFEST_KEY_HARDLINK);
use constant MANIFEST_KEY_ARCHIVE_CHECK                             => 'option-archive-check';
    push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK);
use constant MANIFEST_KEY_ARCHIVE_COPY                              => 'option-archive-copy';
    push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY);
use constant MANIFEST_KEY_BUFFER_SIZE                               => 'option-buffer-size';
    push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE);
use constant MANIFEST_KEY_CHECKSUM_PAGE                             => 'option-checksum-page';
    push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE);
use constant MANIFEST_KEY_COMPRESS                                  => 'option-compress';
    push @EXPORT, qw(MANIFEST_KEY_COMPRESS);
use constant MANIFEST_KEY_COMPRESS_TYPE                             => 'option-compress-type';
    push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE);
use constant MANIFEST_KEY_COMPRESS_LEVEL                            => 'option-compress-level';
    push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL);
use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK                    => 'option-compress-level-network';
    push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK);
use constant MANIFEST_KEY_ONLINE                                    => 'option-online';
    push @EXPORT, qw(MANIFEST_KEY_ONLINE);
use constant MANIFEST_KEY_DELTA                                     => 'option-delta';
    push @EXPORT, qw(MANIFEST_KEY_DELTA);
use constant MANIFEST_KEY_PROCESS_MAX                               => 'option-process-max';
    push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX);

# Information about the database that was backed up
use constant MANIFEST_KEY_DB_ID                                     => 'db-id';
    push @EXPORT, qw(MANIFEST_KEY_DB_ID);
use constant MANIFEST_KEY_SYSTEM_ID                                 => 'db-system-id';
    push @EXPORT, qw(MANIFEST_KEY_SYSTEM_ID);
use constant MANIFEST_KEY_CATALOG                                   => 'db-catalog-version';
    push @EXPORT, qw(MANIFEST_KEY_CATALOG);
use constant MANIFEST_KEY_CONTROL                                   => 'db-control-version';
    push @EXPORT, qw(MANIFEST_KEY_CONTROL);
use constant MANIFEST_KEY_DB_LAST_SYSTEM_ID                         => 'db-last-system-id';
    push @EXPORT, qw(MANIFEST_KEY_DB_LAST_SYSTEM_ID);
use constant MANIFEST_KEY_DB_VERSION                                => 'db-version';
    push @EXPORT, qw(MANIFEST_KEY_DB_VERSION);

# Subkeys used for path/file/link info
use constant MANIFEST_SUBKEY_CHECKSUM                               => 'checksum';
    push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM);
use constant MANIFEST_SUBKEY_CHECKSUM_PAGE                          => 'checksum-page';
    push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE);
use constant MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR                    => 'checksum-page-error';
    push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR);
use constant MANIFEST_SUBKEY_DESTINATION                            => 'destination';
    push @EXPORT, qw(MANIFEST_SUBKEY_DESTINATION);
use constant MANIFEST_SUBKEY_FILE                                   => 'file';
    push @EXPORT, qw(MANIFEST_SUBKEY_FILE);
use constant MANIFEST_SUBKEY_FUTURE                                 => 'future';
    push @EXPORT, qw(MANIFEST_SUBKEY_FUTURE);
use constant MANIFEST_SUBKEY_GROUP                                  => 'group';
    push @EXPORT, qw(MANIFEST_SUBKEY_GROUP);
use constant MANIFEST_SUBKEY_PRIMARY                                => 'mas'.'ter';
    push @EXPORT, qw(MANIFEST_SUBKEY_PRIMARY);
use constant MANIFEST_SUBKEY_MODE                                   => 'mode';
    push @EXPORT, qw(MANIFEST_SUBKEY_MODE);
use constant MANIFEST_SUBKEY_TIMESTAMP                              => 'timestamp';
    push @EXPORT, qw(MANIFEST_SUBKEY_TIMESTAMP);
use constant MANIFEST_SUBKEY_TYPE                                   => 'type';
    push @EXPORT, qw(MANIFEST_SUBKEY_TYPE);
use constant MANIFEST_SUBKEY_PATH                                   => 'path';
    push @EXPORT, qw(MANIFEST_SUBKEY_PATH);
use constant MANIFEST_SUBKEY_REFERENCE                              => 'reference';
    push @EXPORT, qw(MANIFEST_SUBKEY_REFERENCE);
use constant MANIFEST_SUBKEY_REPO_SIZE                              => 'repo-size';
    push @EXPORT, qw(MANIFEST_SUBKEY_REPO_SIZE);
use constant MANIFEST_SUBKEY_SIZE                                   => 'size';
    push @EXPORT, qw(MANIFEST_SUBKEY_SIZE);
use constant MANIFEST_SUBKEY_TABLESPACE_ID                          => 'tablespace-id';
    push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_ID);
use constant MANIFEST_SUBKEY_TABLESPACE_NAME                        => 'tablespace-name';
    push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_NAME);
use constant MANIFEST_SUBKEY_USER                                   => 'user';
    push @EXPORT, qw(MANIFEST_SUBKEY_USER);

####################################################################################################################################
# Database locations for important files/paths
####################################################################################################################################
use constant DB_PATH_ARCHIVESTATUS                                  => 'archive_status';
    push @EXPORT, qw(DB_PATH_ARCHIVESTATUS);
use constant DB_PATH_BASE                                           => 'base';
    push @EXPORT, qw(DB_PATH_BASE);
use constant DB_PATH_GLOBAL                                         => 'global';
    push @EXPORT, qw(DB_PATH_GLOBAL);
use constant DB_PATH_PGDYNSHMEM                                     => 'pg_dynshmem';
    push @EXPORT, qw(DB_PATH_PGDYNSHMEM);
use constant DB_PATH_PGMULTIXACT                                    => 'pg_multixact';
    push @EXPORT, qw(DB_PATH_PGMULTIXACT);
use constant DB_PATH_PGNOTIFY                                       => 'pg_notify';
    push @EXPORT, qw(DB_PATH_PGNOTIFY);
use constant DB_PATH_PGREPLSLOT                                     => 'pg_replslot';
    push @EXPORT, qw(DB_PATH_PGREPLSLOT);
use constant DB_PATH_PGSERIAL                                       => 'pg_serial';
    push @EXPORT, qw(DB_PATH_PGSERIAL);
use constant DB_PATH_PGSNAPSHOTS                                    => 'pg_snapshots';
    push @EXPORT, qw(DB_PATH_PGSNAPSHOTS);
use constant DB_PATH_PGSTATTMP                                      => 'pg_stat_tmp';
    push @EXPORT, qw(DB_PATH_PGSTATTMP);
use constant DB_PATH_PGSUBTRANS                                     => 'pg_subtrans';
    push @EXPORT, qw(DB_PATH_PGSUBTRANS);
use constant DB_PATH_PGTBLSPC                                       => 'pg_tblspc';
    push @EXPORT, qw(DB_PATH_PGTBLSPC);

use constant DB_FILE_BACKUPLABEL                                    => 'backup_label';
    push @EXPORT, qw(DB_FILE_BACKUPLABEL);
use constant DB_FILE_BACKUPLABELOLD                                 => DB_FILE_BACKUPLABEL . '.old';
    push @EXPORT, qw(DB_FILE_BACKUPLABELOLD);
use constant DB_FILE_PGCONTROL                                      => DB_PATH_GLOBAL . '/pg_control';
    push @EXPORT, qw(DB_FILE_PGCONTROL);
use constant DB_FILE_PGFILENODEMAP                                  => 'pg_filenode.map';
    push @EXPORT, qw(DB_FILE_PGFILENODEMAP);
use constant DB_FILE_PGINTERNALINIT                                 => 'pg_internal.init';
    push @EXPORT, qw(DB_FILE_PGINTERNALINIT);
use constant DB_FILE_PGVERSION                                      => 'PG_VERSION';
    push @EXPORT, qw(DB_FILE_PGVERSION);
use constant DB_FILE_POSTGRESQLAUTOCONFTMP                          => 'postgresql.auto.conf.tmp';
    push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP);
use constant DB_FILE_POSTMASTEROPTS                                 => 'postmaster.opts';
    push @EXPORT, qw(DB_FILE_POSTMASTEROPTS);
use constant DB_FILE_POSTMASTERPID                                  => 'postmaster.pid';
    push @EXPORT, qw(DB_FILE_POSTMASTERPID);
use constant DB_FILE_RECOVERYCONF                                   => 'recovery.conf';
    push @EXPORT, qw(DB_FILE_RECOVERYCONF);
use constant DB_FILE_RECOVERYSIGNAL                                 => 'recovery.signal';
    push @EXPORT, qw(DB_FILE_RECOVERYSIGNAL);
use constant DB_FILE_RECOVERYDONE                                   => 'recovery.done';
    push @EXPORT, qw(DB_FILE_RECOVERYDONE);
use constant DB_FILE_STANDBYSIGNAL                                  => 'standby.signal';
    push @EXPORT, qw(DB_FILE_STANDBYSIGNAL);
use constant DB_FILE_TABLESPACEMAP                                  => 'tablespace_map';
    push @EXPORT, qw(DB_FILE_TABLESPACEMAP);

use constant DB_FILE_PREFIX_TMP                                     => 'pgsql_tmp';
    push @EXPORT, qw(DB_FILE_PREFIX_TMP);

####################################################################################################################################
# Manifest locations for important files/paths
####################################################################################################################################
use constant MANIFEST_PATH_BASE                                     => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE;
    push @EXPORT, qw(MANIFEST_PATH_BASE);
use constant MANIFEST_PATH_GLOBAL                                   => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_GLOBAL;
    push @EXPORT, qw(MANIFEST_PATH_GLOBAL);
use constant MANIFEST_PATH_PGDYNSHMEM                               => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM;
    push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM);
use constant MANIFEST_PATH_PGMULTIXACT                              => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGMULTIXACT;
    push @EXPORT, qw(MANIFEST_PATH_PGMULTIXACT);
use constant MANIFEST_PATH_PGNOTIFY                                 => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY;
    push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY);
use constant MANIFEST_PATH_PGREPLSLOT                               => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT;
    push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT);
use constant MANIFEST_PATH_PGSERIAL                                 => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL;
    push @EXPORT, qw(MANIFEST_PATH_PGSERIAL);
use constant MANIFEST_PATH_PGSNAPSHOTS                              => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS;
    push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS);
use constant MANIFEST_PATH_PGSTATTMP                                => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP;
    push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP);
use constant MANIFEST_PATH_PGSUBTRANS                               => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS;
    push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS);
use constant MANIFEST_PATH_PGTBLSPC                                 => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC;
    push @EXPORT, qw(MANIFEST_PATH_PGTBLSPC);

use constant MANIFEST_FILE_BACKUPLABEL                              => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL;
    push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL);
use constant MANIFEST_FILE_BACKUPLABELOLD                           => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD;
    push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD);
use constant MANIFEST_FILE_PGCONTROL                                => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL;
    push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP                    => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP;
    push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
use constant MANIFEST_FILE_POSTMASTEROPTS                           => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTEROPTS;
    push @EXPORT, qw(MANIFEST_FILE_POSTMASTEROPTS);
use constant MANIFEST_FILE_POSTMASTERPID                            => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTERPID;
    push @EXPORT, qw(MANIFEST_FILE_POSTMASTERPID);
use constant MANIFEST_FILE_RECOVERYCONF                             => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF;
    push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF);
use constant MANIFEST_FILE_RECOVERYSIGNAL                           => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYSIGNAL;
    push @EXPORT, qw(MANIFEST_FILE_RECOVERYSIGNAL);
use constant MANIFEST_FILE_RECOVERYDONE                             => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE;
    push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE);
use constant MANIFEST_FILE_STANDBYSIGNAL                            => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_STANDBYSIGNAL;
    push @EXPORT, qw(MANIFEST_FILE_STANDBYSIGNAL);
use constant MANIFEST_FILE_TABLESPACEMAP                            => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP;
    push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP);

####################################################################################################################################
# Minimum ID for a user object in postgres
####################################################################################################################################
use constant DB_USER_OBJECT_MINIMUM_ID                              => 16384;
    push @EXPORT, qw(DB_USER_OBJECT_MINIMUM_ID);

####################################################################################################################################
# new
####################################################################################################################################
sub new
{
    my $class = shift;

    # Assign function parameters, defaults, and log debug info
    my
    (
        $strOperation,
        $strFileName,
        $bLoad,
        $oStorage,
        $strDbVersion,
        $iDbCatalogVersion,
        $strCipherPass,                                             # Passphrase to open the manifest if encrypted
        $strCipherPassSub,                                          # Passphrase to encrypt the backup files
    ) =
        logDebugParam
        (
            __PACKAGE__ . '->new', \@_,
            {name => 'strFileName', trace => true},
            {name => 'bLoad', optional => true, default => true, trace => true},
            {name => 'oStorage', optional => true, default => storageRepo(), trace => true},
            {name => 'strDbVersion', optional => true, trace => true},
            {name => 'iDbCatalogVersion', optional => true, trace => true},
            {name => 'strCipherPass', optional => true, redact => true},
            {name => 'strCipherPassSub', optional => true, redact => true},
        );

    # Init object and store variables
    my $self = $class->SUPER::new(
        $oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub});

    # If manifest not loaded from a file then the db version and catalog version must be set
    if (!$bLoad)
    {
        if (!(defined($strDbVersion) && defined($iDbCatalogVersion)))
        {
            confess &log(ASSERT, 'strDbVersion and iDbCatalogVersion must be provided with bLoad = false');
        }

        # Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one
        $self->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, $strDbVersion . '');
        $self->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, $iDbCatalogVersion);
    }

    # Mark the manifest as built if it was loaded from a file
    $self->{bBuilt} = $bLoad;

    # Return from function and log return values if any
    return logDebugReturn
    (
        $strOperation,
        {name => 'self', value => $self}
    );
}

####################################################################################################################################
# save
#
# Save the manifest.
####################################################################################################################################
sub save
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my ($strOperation) = logDebugParam(__PACKAGE__ . '->save');

    # Call inherited save
    $self->SUPER::save();

    # Return from function and log return values if any
    return logDebugReturn($strOperation);
}

####################################################################################################################################
# get
#
# Get a value.
####################################################################################################################################
sub get
{
    my $self = shift;
    my $strSection = shift;
    my $strKey = shift;
    my $strSubKey = shift;
    my $bRequired = shift;
    my $oDefault = shift;

    my $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, false);

    if (!defined($oValue) && defined($strKey) && defined($strSubKey) &&
        ($strSection eq MANIFEST_SECTION_TARGET_FILE || $strSection eq MANIFEST_SECTION_TARGET_PATH ||
         $strSection eq MANIFEST_SECTION_TARGET_LINK) &&
        ($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP ||
         $strSubKey eq MANIFEST_SUBKEY_MODE || $strSubKey eq MANIFEST_SUBKEY_PRIMARY) &&
        $self->test($strSection, $strKey))
    {
        $oValue = $self->SUPER::get("${strSection}:default", $strSubKey, undef, $bRequired, $oDefault);
    }
    else
    {
        $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, $bRequired, $oDefault);
    }

    return $oValue;
}

####################################################################################################################################
# boolGet
#
# Get a numeric value.
####################################################################################################################################
sub boolGet
{
    my $self = shift;
    my $strSection = shift;
    my $strValue = shift;
    my $strSubValue = shift;
    my $bRequired = shift;
    my $bDefault = shift;

    return $self->get($strSection, $strValue, $strSubValue, $bRequired,
                      defined($bDefault) ? ($bDefault ? INI_TRUE : INI_FALSE) : undef) ? true : false;
}

####################################################################################################################################
# numericGet
#
# Get a numeric value.
####################################################################################################################################
sub numericGet
{
    my $self = shift;
    my $strSection = shift;
    my $strValue = shift;
    my $strSubValue = shift;
    my $bRequired = shift;
    my $nDefault = shift;

    return $self->get($strSection, $strValue, $strSubValue, $bRequired,
                      defined($nDefault) ? $nDefault + 0 : undef) + 0;
}

####################################################################################################################################
# tablespacePathGet
#
# Get the unique path assigned by Postgres for the tablespace.
####################################################################################################################################
sub tablespacePathGet
{
    my $self = shift;

    return('PG_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) .
           '_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG));
}

####################################################################################################################################
# dbPathGet
#
# Convert a repo path to where the file actually belongs in the db.
####################################################################################################################################
sub dbPathGet
{
    my $self = shift;
    my $strDbPath = shift;
    my $strFile = shift;

    my $strDbFile = defined($strDbPath) ? "${strDbPath}/" : '';

    if (index($strFile, MANIFEST_TARGET_PGDATA . '/') == 0)
    {
        $strDbFile .= substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1);
    }
    else
    {
        $strDbFile .= $strFile;
    }

    return $strDbFile;
}

####################################################################################################################################
# repoPathGet
#
# Convert a database path to where to file is located in the repo.
####################################################################################################################################
sub repoPathGet
{
    my $self = shift;
    my $strTarget = shift;
    my $strFile = shift;

    my $strRepoFile = $strTarget;

    if ($self->isTargetTablespace($strTarget) &&
        ($self->dbVersion() >= PG_VERSION_90))
    {
        $strRepoFile .= '/' . $self->tablespacePathGet();
    }

    if (defined($strFile))
    {
        $strRepoFile .= "/${strFile}";
    }

    return $strRepoFile;
}

####################################################################################################################################
# isTargetValid
#
# Determine if a target is valid.
####################################################################################################################################
sub isTargetValid
{
    my $self = shift;
    my $strTarget = shift;
    my $bError = shift;

    if (!defined($strTarget))
    {
        confess &log(ASSERT, 'target is not defined');
    }

    if (!$self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget))
    {
        if (defined($bError) && $bError)
        {
            confess &log(ASSERT, "${strTarget} is not a valid target");
        }

        return false;
    }

    return true;
}

####################################################################################################################################
# isTargetLink
#
# Determine if a target is a link.
####################################################################################################################################
sub isTargetLink
{
    my $self = shift;
    my $strTarget = shift;

    $self->isTargetValid($strTarget, true);

    return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TYPE, MANIFEST_VALUE_LINK);
}

####################################################################################################################################
# isTargetFile
#
# Determine if a target is a file link.
####################################################################################################################################
sub isTargetFile
{
    my $self = shift;
    my $strTarget = shift;

    $self->isTargetValid($strTarget, true);

    return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE);
}

####################################################################################################################################
# isTargetTablespace
#
# Determine if a target is a tablespace.
####################################################################################################################################
sub isTargetTablespace
{
    my $self = shift;
    my $strTarget = shift;

    $self->isTargetValid($strTarget, true);

    return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TABLESPACE_ID);
}

####################################################################################################################################
# checkDelta
#
# Determine if the delta option should be enabled. Only called if delta has not yet been enabled.
####################################################################################################################################
sub checkDelta
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my
    (
        $strOperation,
        $strLastBackupSource,
        $bOnlineSame,
        $strTimelineCurrent,
        $strTimelineLast,
    ) =
        logDebugParam
        (
            __PACKAGE__ . '->checkDelta', \@_,
            {name => 'strLastBackupSource'},
            {name => 'bOnlineSame'},
            {name => 'strTimelineCurrent', required => false},
            {name => 'strTimelineLast', required => false},
        );

    my $bDelta = false;

    # Determine if a timeline switch has occurred
    if (defined($strTimelineLast) && defined($strTimelineCurrent))
    {
        # If there is a prior backup, check if a timeline switch has occurred since then
        if ($strTimelineLast ne $strTimelineCurrent)
        {
            &log(WARN, "a timeline switch has occurred since the ${strLastBackupSource} backup, enabling delta checksum");
            $bDelta = true;
        }
    }

    # If delta was not set above and there is a change in the online option, then set delta option
    if (!$bDelta && !$bOnlineSame)
    {
        &log(WARN, "the online option has changed since the ${strLastBackupSource} backup, enabling delta checksum");
        $bDelta = true;
    }

    # Return from function and log return values if any
    return logDebugReturn
    (
        $strOperation,
        {name => 'bDelta', value => $bDelta, trace => true},
    );
}

####################################################################################################################################
# checkDeltaFile
#
# Determine if the delta option should be enabled. Only called if delta has not yet been enabled.
####################################################################################################################################
sub checkDeltaFile
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my
    (
        $strOperation,
        $stryFileList,
        $oPriorManifest,
        $lTimeBegin,
    ) =
        logDebugParam
        (
            __PACKAGE__ . '->checkDeltaFile', \@_,
            {name => 'stryFileList'},
            {name => 'oPriorManifest', required => false},
            {name => 'lTimeBegin', required => false},
        );

    my $bDelta = false;

    # Loop though all files
    foreach my $strName (@{$stryFileList})
    {
        # If $lTimeBegin is defined, then this is not an aborted manifest so check if modification time is in the future (in this
        # backup OR the last backup) then enable delta and exit
        if (defined($lTimeBegin) &&
            ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin ||
            (defined($oPriorManifest) &&
             $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))))
        {
            &log(WARN, "file $strName has timestamp in the future, enabling delta checksum");
            $bDelta = true;
            last;
        }

        # If the time on the file is earlier than the last manifest time or the size is different but the timestamp is the
        # same, then enable delta and exit
        if (defined($oPriorManifest) && $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) &&
            ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) <
            $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ||
            ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) !=
            $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) &&
            $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ==
            $oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP))))
        {
            &log(WARN, "file $strName timestamp in the past or size changed but timestamp did not, enabling delta checksum");
            $bDelta = true;
            last;
        }
    }

    # Return from function and log return values if any
    return logDebugReturn
    (
        $strOperation,
        {name => 'bDelta', value => $bDelta, trace => true},
    );
}

####################################################################################################################################
# build
#
# Build the manifest object.
####################################################################################################################################
sub build
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my
    (
        $strOperation,
        $oStorageDbPrimary,
        $strPath,
        $oLastManifest,
        $bOnline,
        $bDelta,
        $hTablespaceMap,
        $hDatabaseMap,
        $rhExclude,
        $strTimelineCurrent,
        $strTimelineLast,
        $strLevel,
        $bTablespace,
        $strParentPath,
        $strFilter,
        $iLevel,
    ) =
        logDebugParam
        (
            __PACKAGE__ . '->build', \@_,
            {name => 'oStorageDbPrimary'},
            {name => 'strPath'},
            {name => 'oLastManifest', required => false},
            {name => 'bOnline'},
            {name => 'bDelta'},
            {name => 'hTablespaceMap', required => false},
            {name => 'hDatabaseMap', required => false},
            {name => 'rhExclude', required => false},
            {name => 'strTimelineCurrent', required => false},
            {name => 'strTimelineLast', required => false},
            {name => 'strLevel', required => false},
            {name => 'bTablespace', required => false},
            {name => 'strParentPath', required => false},
            {name => 'strFilter', required => false},
            {name => 'iLevel', required => false, default => 0},
        );

    # Limit recursion to something reasonable (if more then we are very likely in a link loop)
    if ($iLevel >= 16)
    {
        confess &log(
            ERROR,
            "recursion in manifest build exceeds depth of ${iLevel}: ${strLevel}\n" .
                'HINT: is there a link loop in $PGDATA?',
            ERROR_FORMAT);
    }

    if (!defined($strLevel))
    {
        # Don't allow the manifest to be built more than once
        if ($self->{bBuilt})
        {
            confess &log(ASSERT, "manifest has already been built");
        }

        $self->{bBuilt} = true;

        # Set initial level
        $strLevel = MANIFEST_TARGET_PGDATA;

        # If not online then build the tablespace map from pg_tblspc path
        if (!$bOnline && !defined($hTablespaceMap))
        {
            my $hTablespaceManifest = $oStorageDbPrimary->manifest($strPath . '/' . DB_PATH_PGTBLSPC);
            $hTablespaceMap = {};

            foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest})))
            {
                if ($strOid eq '.' or $strOid eq '..')
                {
                    next;
                }

                logDebugMisc($strOperation, "found tablespace ${strOid} in offline mode");

                $hTablespaceMap->{$strOid} = "ts${strOid}";
            }
        }

        # If there is a last manifest, then check to see if delta checksum should be enabled
        if (defined($oLastManifest) && !$bDelta)
        {
            $bDelta = $self->checkDelta(
                'last', $oLastManifest->boolTest(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, $bOnline),
                $strTimelineCurrent, $strTimelineLast);
        }
    }

    $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, $strPath);
    $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TYPE,
               $strLevel eq MANIFEST_TARGET_PGDATA ? MANIFEST_VALUE_PATH : MANIFEST_VALUE_LINK);

    if ($bTablespace)
    {
        my $iTablespaceId = (split('\/', $strLevel))[1];

        if (!defined($hTablespaceMap->{$iTablespaceId}))
        {
            confess &log(ASSERT, "tablespace with oid ${iTablespaceId} not found in tablespace map\n" .
                                 "HINT: was a tablespace created or dropped during the backup?");
        }

        $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_ID, $iTablespaceId);
        $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_NAME,
                   $hTablespaceMap->{$iTablespaceId});
    }

    if (index($strPath, '/') != 0)
    {
        if (!defined($strParentPath))
        {
            confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified");
        }

        $strPath = $oStorageDbPrimary->pathAbsolute($strParentPath, $strPath);
    }

    # Get the manifest for this level
    my $hManifest = $oStorageDbPrimary->manifest($strPath, {strFilter => $strFilter});
    my $strManifestType = MANIFEST_VALUE_LINK;

    # Loop though all paths/files/links in the manifest
    foreach my $strName (sort(CORE::keys(%{$hManifest})))
    {
        my $strFile = $strLevel;

        if ($strName ne '.')
        {
            if ($strManifestType eq MANIFEST_VALUE_LINK && $hManifest->{$strName}{type} eq 'l')
            {
                confess &log(ERROR, 'link \'' .
                    $self->dbPathGet(
                        $self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH), $strLevel) .
                        '\' -> \'' . $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH) .
                        '\' cannot reference another link', ERROR_LINK_DESTINATION);
            }

            if ($strManifestType eq MANIFEST_VALUE_LINK)
            {
                $strFile = dirname($strFile);
                $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH,
                           dirname($self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH)));
                $self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_FILE, $strName);
            }

            $strFile .= "/${strName}";
        }
        else
        {
            $strManifestType = MANIFEST_VALUE_PATH;
        }

        # Skip wal directory when doing an online backup.  WAL will be restored from the archive or stored in the wal directory at
        # the end of the backup if the archive-copy option is set.
        next if ($bOnline && $strFile =~ (qw{^} . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . '\/') &&
                 $strFile !~ ('^' . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS . '$'));

        # Skip all directories and files that start with pgsql_tmp.  The files are removed when the server is restarted and the
        # directories are recreated.
        next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP);

        # Skip pg_dynshmem/* since these files cannot be reused on recovery
        next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $self->dbVersion() >= PG_VERSION_94;

        # Skip pg_notify/* since these files cannot be reused on recovery
        next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/') && $self->dbVersion() >= PG_VERSION_90;

        # Skip pg_replslot/* since these files are generally not useful after a restore
        next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $self->dbVersion() >= PG_VERSION_94;

        # Skip pg_serial/* since these files are reset
        next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/') && $self->dbVersion() >= PG_VERSION_91;

        # Skip pg_snapshots/* since these files cannot be reused on recovery
        next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/') && $self->dbVersion() >= PG_VERSION_92;

        # Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created
        # there.
        next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/') && $self->dbVersion() >= PG_VERSION_84;

        # Skip pg_subtrans/* since these files are reset
        next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/');

        # Skip pg_internal.init since it is recreated on startup
        next if $strFile =~ (DB_FILE_PGINTERNALINIT . '$');

        # Skip recovery files
        if ($self->dbVersion() >= PG_VERSION_12)
        {
            next if ($strFile eq MANIFEST_FILE_RECOVERYSIGNAL || $strFile eq MANIFEST_FILE_STANDBYSIGNAL);
        }
        else
        {
            next if ($strFile eq MANIFEST_FILE_RECOVERYDONE || $strFile eq MANIFEST_FILE_RECOVERYCONF);
        }

        # Skip ignored files
        if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP ||      # postgresql.auto.conf.tmp - temp file for safe writes
            $strFile eq MANIFEST_FILE_BACKUPLABELOLD ||             # backup_label.old - old backup labels are not useful
            $strFile eq MANIFEST_FILE_POSTMASTEROPTS ||             # postmaster.opts - not useful for backup
            $strFile eq MANIFEST_FILE_POSTMASTERPID)                # postmaster.pid - to avoid confusing postgres after restore
        {
            next;
        }

        # If version is greater than 9.0, check for files to exclude
        if ($self->dbVersion() >= PG_VERSION_90 && $hManifest->{$strName}{type} eq 'f')
        {
            # Get the directory name from the manifest; it will be used later to search for existence in the keys
            my $strDir = dirname($strName);

            # If it is a database data directory (base or tablespace) then check for files to skip
            if ($strDir =~ '^base\/[0-9]+$' ||
                $strDir =~ ('^' . $self->tablespacePathGet() . '\/[0-9]+$'))
            {
                # Get just the filename
                my $strBaseName = basename($strName);

                # Skip temp tables (lower t followed by numbers underscore numbers and a dot (segment) or underscore (fork) and/or
                # segment, e.g. t1234_123, t1234_123.1, t1234_123_vm, t1234_123_fsm.1
                if ($strBaseName =~ '^t[0-9]+\_[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$')
                {
                    next;
                }

                # If version is greater than 9.1 then check for unlogged tables to skip
                if ($self->dbVersion() >= PG_VERSION_91)
                {
                    # Exclude all forks for unlogged tables except the init fork (numbers underscore init and optional dot segment)
                    if ($strBaseName =~ '^[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$')
                    {
                        # Get the filenode (OID)
                        my ($strFileNode) = $strBaseName =~ '^(\d+)';

                        # Add _init to the OID to see if this is an unlogged object
                        $strFileNode = $strDir. "/" . $strFileNode . "_init";

                        # If key exists in manifest then skip
                        if (exists($hManifest->{$strFileNode}) && $hManifest->{$strFileNode}{type} eq 'f')
                        {
                            next;
                        }
                    }
                }
            }
        }

        # Exclude files requested by the user
        if (defined($rhExclude))
        {
            # Exclusions are based on the name of the file relative to PGDATA
            my $strPgFile = $self->dbPathGet(undef, $strFile);
            my $bExclude = false;

            # Iterate through exclusions
            foreach my $strExclude (sort(keys(%{$rhExclude})))
            {
                # If the exclusion ends in / then we must do a prefix match
                if ($strExclude =~ /\/$/)
                {
                    if (index($strPgFile, $strExclude) == 0)
                    {
                        $bExclude = true;
                    }
                }
                # Else an exact match or a prefix match with / appended is required
                elsif ($strPgFile eq $strExclude || index($strPgFile, "${strExclude}/") == 0)
                {
                    $bExclude = true;
                }

                # Log everything that gets excluded at a high level so it will hopefully be seen if wrong
                if ($bExclude)
                {
                    &log(INFO, "exclude ${strPgFile} from backup using '${strExclude}' exclusion");
                    last;
                }
            }

            # Skip the file if it was excluded
            next if $bExclude;
        }

        my $cType = $hManifest->{$strName}{type};
        my $strSection = MANIFEST_SECTION_TARGET_PATH;

        if ($cType eq 'f')
        {
            $strSection = MANIFEST_SECTION_TARGET_FILE;
        }
        elsif ($cType eq 'l')
        {
            $strSection = MANIFEST_SECTION_TARGET_LINK;
        }
        elsif ($cType ne 'd')
        {
            &log(WARN, "exclude special file '" . $self->dbPathGet(undef, $strFile) . "' from backup");
            next;
        }

        # Make sure that DB_PATH_PGTBLSPC contains only absolute links that do not point inside PGDATA
        my $bTablespace = false;

        if (index($strName, DB_PATH_PGTBLSPC . '/') == 0 && $strLevel eq MANIFEST_TARGET_PGDATA)
        {
            $bTablespace = true;
            $strFile = MANIFEST_TARGET_PGDATA . '/' . $strName;

            # Check for files in DB_PATH_PGTBLSPC that are not links
            if ($hManifest->{$strName}{type} ne 'l')
            {
                confess &log(ERROR, "${strName} is not a symlink - " . DB_PATH_PGTBLSPC . ' should contain only symlinks',
                             ERROR_LINK_EXPECTED);
            }

            # Check for tablespaces in PGDATA
            if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 ||
                (index($hManifest->{$strName}{link_destination}, '/') != 0 &&
                 index($oStorageDbPrimary->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC,
                       $hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0))
            {
                confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} .
                             ' destination must not be in $PGDATA', ERROR_LINK_DESTINATION);
            }
        }

        # User and group required for all types
        if (defined($hManifest->{$strName}{user}))
        {
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_USER, $hManifest->{$strName}{user});
        }
        else
        {
            $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_USER, false);
        }

        if (defined($hManifest->{$strName}{group}))
        {
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $hManifest->{$strName}{group});
        }
        else
        {
            $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_GROUP, false);
        }

        # Mode for required file and path type only
        if ($cType eq 'f' || $cType eq 'd')
        {
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_MODE, $hManifest->{$strName}{mode});
        }

        # Modification time and size required for file type only
        if ($cType eq 'f')
        {
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_TIMESTAMP,
                       $hManifest->{$strName}{modification_time} + 0);
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_SIZE, $hManifest->{$strName}{size} + 0);
            $self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_PRIMARY,
                ($strFile eq MANIFEST_FILE_PGCONTROL || $self->isPrimaryFile($strFile)));
        }

        # Link destination required for link type only
        if ($cType eq 'l')
        {
            my $strLinkDestination = $hManifest->{$strName}{link_destination};
            $self->set($strSection, $strFile, MANIFEST_SUBKEY_DESTINATION, $strLinkDestination);

            # If this is a tablespace then set the filter to use for the next level
            my $strFilter;

            if ($bTablespace)
            {
                # Only versions >= 9.0  have the special top-level tablespace path.  Below 9.0 the database files are stored
                # directly in the path referenced by the symlink.
                if ($self->dbVersion() >= PG_VERSION_90)
                {
                    $strFilter = $self->tablespacePathGet();
                }

                $self->set(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGTBLSPC, undef,
                           $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA));

                # PGDATA prefix was only needed for the link so strip it off before recursing
                $strFile = substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1);
            }

            $bDelta = $self->build(
                $oStorageDbPrimary, $strLinkDestination, undef, $bOnline, $bDelta, $hTablespaceMap, $hDatabaseMap, $rhExclude,
                undef, undef, $strFile, $bTablespace, dirname("${strPath}/${strName}"), $strFilter, $iLevel + 1);
        }
    }

    # If this is the base level then do post-processing
    if ($strLevel eq MANIFEST_TARGET_PGDATA)
    {
        my $bTimeInFuture = false;

        # Wait for the remainder of the second when doing online backups.  This is done because most filesystems only have a one
        # second resolution and Postgres will still be modifying files during the second that the manifest is built and this could
        # lead to an invalid diff/incr backup later when using timestamps to determine which files have changed.  Offline backups do
        # not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be
        # consistent anyway and the one-second resolution problem is the least of our worries).
        my $lTimeBegin = waitRemainder($bOnline);

        if (defined($oLastManifest))
        {
            $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef,
                       $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
        }

        # Store database map information when provided during an online backup.
        foreach my $strDbName (sort(keys(%{$hDatabaseMap})))
        {
            $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_ID,
                              $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_ID});
            $self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_LAST_SYSTEM_ID,
                              $hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID});
        }

        # Determine if delta checksum should be enabled
        if (!$bDelta)
        {
            my @stryFileList = $self->keys(MANIFEST_SECTION_TARGET_FILE);

            if (@stryFileList)
            {
                $bDelta = $self->checkDeltaFile(\@stryFileList, $oLastManifest, $lTimeBegin);
            }
        }

        # Loop though all files
        foreach my $strName ($self->keys(MANIFEST_SECTION_TARGET_FILE))
        {
            # If modification time is in the future (in this backup OR the last backup) set warning flag and do not
            # allow a reference
            if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin ||
                (defined($oLastManifest) &&
                 $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))
            {
                $bTimeInFuture = true;

                # Only mark as future if still in the future in the current backup
                if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin)
                {
                    $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y');
                }
            }
            # Else check if the size and timestamp match OR if the size matches and the delta option is set, then keep the file.
            # In the latter case, if there had been a timestamp change then rather than removing and recopying the file, the file
            # will be tested in backupFile to see if the db/repo checksum still matches: if so, it is not necessary to recopy,
            # else it will need to be copied to the new backup. For zero sized files, the reference will be set and copying
            # will be skipped later.
            elsif (defined($oLastManifest) && $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) &&
                   $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) ==
                       $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) &&
                   ($bDelta || ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == 0 ||
                   $self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ==
                       $oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP))))
            {
                # Copy reference from previous backup if possible
                if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE))
                {
                    $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE,
                        $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE));
                }
                # Otherwise the reference is to the previous backup
                else
                {
                    $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE,
                        $oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
                }

                # Copy the checksum from previous manifest (if it exists - zero sized files don't have checksums)
                if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM))
                {
                    $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM,
                               $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM));
                }

                # Copy repo size from the previous manifest (if it exists)
                if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE))
                {
                    $self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE,
                        $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE));
                }

                # Copy primary flag from the previous manifest (if it exists)
                if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_PRIMARY))
                {
                    $self->set(
                        MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_PRIMARY,
                        $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_PRIMARY));
                }

                # Copy checksum page from the previous manifest (if it exists)
                my $bChecksumPage = $oLastManifest->get(
                    MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, false);

                if (defined($bChecksumPage))
                {
                    $self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage);

                    if (!$bChecksumPage &&
                        $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR))
                    {
                        $self->set(
                            MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,
                            $oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR));
                    }
                }
            }
        }

        # Warn if any files in the current backup are in the future
        if ($bTimeInFuture)
        {
            &log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions");
        }

        # Record the time when copying will start
        $self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $lTimeBegin + ($bOnline ? 1 : 0));

        # Build default sections
        $self->buildDefault();
    }

    # Return from function and log return values if any
    return logDebugReturn
    (
        $strOperation,
        {name => 'bDelta', value => $bDelta, trace => true},
    );
}

####################################################################################################################################
# fileAdd
#
# Add files to the manifest that were generated after the initial manifest build, e.g. backup_label, tablespace_map, and copied WAL
# files.  Since the files were not in the original cluster the user, group, and mode must be defaulted.
####################################################################################################################################
sub fileAdd
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my
    (
        $strOperation,
        $strManifestFile,
        $lModificationTime,
        $lSize,
        $strChecksum,
        $bPrimary,
    ) =
        logDebugParam
        (
            __PACKAGE__ . '->fileAdd', \@_,
            {name => 'strManifestFile'},
            {name => 'lModificationTime'},
            {name => 'lSize'},
            {name => 'lChecksum'},
            {name => 'bPrimary'},
        );

    # Set manifest values
    if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER) ||
        !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef,
                     $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER)))
    {
        $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_USER,
                   $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER));
    }

    if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP) ||
        !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef,
                     $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP)))
    {
        $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_GROUP,
                   $self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP));
    }

    if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE) ||
        !$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, '0600'))
    {
        $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MODE, '0600');
    }

    $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_TIMESTAMP, $lModificationTime);
    $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_SIZE, $lSize);
    $self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
    $self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_PRIMARY, $bPrimary);
}

####################################################################################################################################
# buildDefault
#
# Builds the default section.
####################################################################################################################################
sub buildDefault
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my ($strOperation) = logDebugParam(__PACKAGE__ . '->buildDefault');

    # Set defaults for subkeys that tend to repeat
    foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK)
    {
        foreach my $strSubKey (&MANIFEST_SUBKEY_USER, &MANIFEST_SUBKEY_GROUP, &MANIFEST_SUBKEY_MODE, &MANIFEST_SUBKEY_PRIMARY)
        {
            # Links don't have a mode so skip
            next if ($strSection eq MANIFEST_SECTION_TARGET_LINK && $strSubKey eq &MANIFEST_SUBKEY_MODE);

            # Only files have the primary subkey
            next if ($strSection ne MANIFEST_SECTION_TARGET_FILE && $strSubKey eq &MANIFEST_SUBKEY_PRIMARY);

            my %oDefault;
            my $iSectionTotal = 0;

            foreach my $strFile ($self->keys($strSection))
            {
                # Don't count false values when subkey in (MANIFEST_SUBKEY_USER, MANIFEST_SUBKEY_GROUP)
                next if (($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP) &&
                         $self->boolTest($strSection, $strFile, $strSubKey, false));

                my $strValue = $self->get($strSection, $strFile, $strSubKey);

                if (defined($oDefault{$strValue}))
                {
                    $oDefault{$strValue}++;
                }
                else
                {
                    $oDefault{$strValue} = 1;
                }

                $iSectionTotal++;
            }

            my $strMaxValue;
            my $iMaxValueTotal = 0;

            foreach my $strValue (sort(keys(%oDefault)))
            {
                if ($oDefault{$strValue} > $iMaxValueTotal)
                {
                    $iMaxValueTotal = $oDefault{$strValue};
                    $strMaxValue = $strValue;
                }
            }

            if (defined($strMaxValue) > 0 && $iMaxValueTotal > $iSectionTotal * MANIFEST_DEFAULT_MATCH_FACTOR)
            {
                if ($strSubKey eq MANIFEST_SUBKEY_PRIMARY)
                {
                    $self->boolSet("${strSection}:default", $strSubKey, undef, $strMaxValue);
                }
                else
                {
                    $self->set("${strSection}:default", $strSubKey, undef, $strMaxValue);
                }

                foreach my $strFile ($self->keys($strSection))
                {
                    if ($self->test($strSection, $strFile, $strSubKey, $strMaxValue))
                    {
                        $self->remove($strSection, $strFile, $strSubKey);
                    }
                }
            }
        }
    }

    # Return from function and log return values if any
    return logDebugReturn($strOperation);
}

####################################################################################################################################
# validate
#
# Checks for any missing values or inconsistencies in the manifest.
####################################################################################################################################
sub validate
{
    my $self = shift;

    # Assign function parameters, defaults, and log debug info
    my ($strOperation) = logDebugParam(__PACKAGE__ . 'validate');

    # Make sure that all files have size and checksum (when size > 0).  Since these values are removed before the backup file copy
    # starts this ensures that all files had results stored in the manifest during the file copy.
    foreach my $strFile ($self->keys(MANIFEST_SECTION_TARGET_FILE))
    {
        # Ensure size is set
        if (!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE))
        {
            confess &log(ASSERT, "manifest subvalue 'size' not set for file '${strFile}'");
        }

        # If size > 0 then checksum must also be set
        if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE) > 0 &&
            !$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_CHECKSUM))
        {
            confess &log(ASSERT, "manifest subvalue 'checksum' not set for file '${strFile}'");
        }
    }

    # Return from function and log return values if any
    return logDebugReturn($strOperation);
}

####################################################################################################################################
# dbVersion - version of PostgreSQL that the manifest is being built for
####################################################################################################################################
sub dbVersion
{
    my $self = shift;

    return $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION);
}

####################################################################################################################################
# xactPath - return the transaction directory based on the PostgreSQL version
####################################################################################################################################
sub xactPath
{
    my $self = shift;

    return $self->dbVersion() >= PG_VERSION_10 ? 'pg_xact' : 'pg_clog';
}

####################################################################################################################################
# walPath - return the wal directory based on the PostgreSQL version
####################################################################################################################################
sub walPath
{
    my $self = shift;

    return $self->dbVersion() >= PG_VERSION_10 ? 'pg_wal' : 'pg_xlog';
}

####################################################################################################################################
# isPrimaryFile
#
# Is this file required to be copied from the primary?
####################################################################################################################################
sub isPrimaryFile
{
    my $self = shift;
    my $strFile = shift;

    return
        $strFile !~ ('^(' . MANIFEST_TARGET_PGDATA . '\/' . '(' . DB_PATH_BASE . '|' . DB_PATH_GLOBAL . '|' .
        $self->xactPath() . '|' . DB_PATH_PGMULTIXACT . ')|' . DB_PATH_PGTBLSPC . ')\/');
}

####################################################################################################################################
# isChecksumPage
#
# Can this file have page checksums in PG >= 9.3?
####################################################################################################################################
sub isChecksumPage
{
    my $strFile = shift;

    if (($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_BASE . '\/[0-9]+\/|^' . MANIFEST_TARGET_PGTBLSPC .
            '\/[0-9]+\/[^\/]+\/[0-9]+\/') &&
            $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . ')$')) ||
        ($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_GLOBAL . '\/') &&
            $strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . '|' .
            DB_FILE_PGCONTROL . ')$')))
    {
        return true;
    }

    return false;
}

push @EXPORT, qw(isChecksumPage);

1;