2015-01-07 17:59:43 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# MANIFEST MODULE
|
|
|
|
####################################################################################################################################
|
2016-04-14 15:30:54 +02:00
|
|
|
package pgBackRest::Manifest;
|
|
|
|
use parent 'pgBackRest::Common::Ini';
|
2015-01-07 17:59:43 +02:00
|
|
|
|
|
|
|
use strict;
|
2015-03-03 07:57:20 +02:00
|
|
|
use warnings FATAL => qw(all);
|
|
|
|
use Carp qw(confess);
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2015-06-14 00:25:49 +02:00
|
|
|
use Exporter qw(import);
|
2015-08-29 20:20:46 +02:00
|
|
|
our @EXPORT = qw();
|
2015-02-01 01:15:08 +02:00
|
|
|
use File::Basename qw(dirname basename);
|
2015-06-14 00:25:49 +02:00
|
|
|
use Time::Local qw(timelocal);
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2016-08-12 04:35:24 +02:00
|
|
|
use pgBackRest::DbVersion;
|
2016-04-14 15:30:54 +02:00
|
|
|
use pgBackRest::Common::Exception;
|
|
|
|
use pgBackRest::Common::Ini;
|
|
|
|
use pgBackRest::Common::Log;
|
2017-06-09 23:51:41 +02:00
|
|
|
use pgBackRest::Common::Wait;
|
|
|
|
use pgBackRest::Storage::Helper;
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2015-03-12 18:15:19 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# File/path constants
|
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant PATH_BACKUP_HISTORY => 'backup.history';
|
|
|
|
push @EXPORT, qw(PATH_BACKUP_HISTORY);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant FILE_MANIFEST => 'backup.manifest';
|
|
|
|
push @EXPORT, qw(FILE_MANIFEST);
|
2017-06-09 23:51:41 +02:00
|
|
|
use constant FILE_MANIFEST_COPY => FILE_MANIFEST . INI_COPY_EXT;
|
|
|
|
push @EXPORT, qw(FILE_MANIFEST_COPY);
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# Default match factor
|
|
|
|
####################################################################################################################################
|
|
|
|
use constant MANIFEST_DEFAULT_MATCH_FACTOR => 0.1;
|
|
|
|
push @EXPORT, qw(MANIFEST_DEFAULT_MATCH_FACTOR);
|
2015-03-12 18:15:19 +02:00
|
|
|
|
2015-01-07 17:59:43 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# MANIFEST Constants
|
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_TARGET_PGDATA => 'pg_data';
|
|
|
|
push @EXPORT, qw(MANIFEST_TARGET_PGDATA);
|
|
|
|
use constant MANIFEST_TARGET_PGTBLSPC => 'pg_tblspc';
|
|
|
|
push @EXPORT, qw(MANIFEST_TARGET_PGTBLSPC);
|
2015-06-14 00:25:49 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_VALUE_PATH => 'path';
|
|
|
|
push @EXPORT, qw(MANIFEST_VALUE_PATH);
|
|
|
|
use constant MANIFEST_VALUE_LINK => 'link';
|
|
|
|
push @EXPORT, qw(MANIFEST_VALUE_LINK);
|
2015-06-14 00:25:49 +02:00
|
|
|
|
|
|
|
# Manifest sections
|
|
|
|
use constant MANIFEST_SECTION_BACKUP => 'backup';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_BACKUP);
|
|
|
|
use constant MANIFEST_SECTION_BACKUP_DB => 'backup:db';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_BACKUP_DB);
|
|
|
|
use constant MANIFEST_SECTION_BACKUP_INFO => 'backup:info';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_BACKUP_INFO);
|
|
|
|
use constant MANIFEST_SECTION_BACKUP_OPTION => 'backup:option';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_BACKUP_OPTION);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SECTION_BACKUP_TARGET => 'backup:target';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_BACKUP_TARGET);
|
2016-05-11 15:21:39 +02:00
|
|
|
use constant MANIFEST_SECTION_DB => 'db';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_DB);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SECTION_TARGET_PATH => 'target:path';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_TARGET_PATH);
|
|
|
|
use constant MANIFEST_SECTION_TARGET_FILE => 'target:file';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_TARGET_FILE);
|
|
|
|
use constant MANIFEST_SECTION_TARGET_LINK => 'target:link';
|
|
|
|
push @EXPORT, qw(MANIFEST_SECTION_TARGET_LINK);
|
2015-06-14 00:25:49 +02:00
|
|
|
|
|
|
|
# Backup metadata required for restores
|
|
|
|
use constant MANIFEST_KEY_ARCHIVE_START => 'backup-archive-start';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_START);
|
|
|
|
use constant MANIFEST_KEY_ARCHIVE_STOP => 'backup-archive-stop';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_STOP);
|
|
|
|
use constant MANIFEST_KEY_LABEL => 'backup-label';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_LABEL);
|
2016-06-24 14:06:20 +02:00
|
|
|
use constant MANIFEST_KEY_LSN_START => 'backup-lsn-start';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_LSN_START);
|
|
|
|
use constant MANIFEST_KEY_LSN_STOP => 'backup-lsn-stop';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_LSN_STOP);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_KEY_PRIOR => 'backup-prior';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_PRIOR);
|
|
|
|
use constant MANIFEST_KEY_TIMESTAMP_COPY_START => 'backup-timestamp-copy-start';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_COPY_START);
|
|
|
|
use constant MANIFEST_KEY_TIMESTAMP_START => 'backup-timestamp-start';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_START);
|
|
|
|
use constant MANIFEST_KEY_TIMESTAMP_STOP => 'backup-timestamp-stop';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_TIMESTAMP_STOP);
|
|
|
|
use constant MANIFEST_KEY_TYPE => 'backup-type';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_TYPE);
|
|
|
|
|
|
|
|
# Options that were set when the backup was made
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_BACKUP_STANDBY => 'option-backup-standby';
|
2016-08-25 17:25:46 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_BACKUP_STANDBY);
|
2018-02-04 01:27:38 +02:00
|
|
|
use constant MANIFEST_KEY_HARDLINK => 'option-hardlink';
|
2015-06-14 00:25:49 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_HARDLINK);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_ARCHIVE_CHECK => 'option-archive-check';
|
2015-06-14 00:25:49 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_CHECK);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_ARCHIVE_COPY => 'option-archive-copy';
|
2015-06-14 00:25:49 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_ARCHIVE_COPY);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_BUFFER_SIZE => 'option-buffer-size';
|
2019-03-10 11:03:52 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_BUFFER_SIZE);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_CHECKSUM_PAGE => 'option-checksum-page';
|
2016-12-13 01:54:07 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_CHECKSUM_PAGE);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_COMPRESS => 'option-compress';
|
2015-06-14 00:25:49 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_COMPRESS);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_COMPRESS_TYPE => 'option-compress-type';
|
2020-03-06 21:41:03 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_TYPE);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_COMPRESS_LEVEL => 'option-compress-level';
|
2019-03-10 11:03:52 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_COMPRESS_LEVEL_NETWORK => 'option-compress-level-network';
|
2019-03-10 11:03:52 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_COMPRESS_LEVEL_NETWORK);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_ONLINE => 'option-online';
|
2016-02-20 02:32:02 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_ONLINE);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_DELTA => 'option-delta';
|
2018-09-19 17:12:45 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_DELTA);
|
2020-03-09 23:41:59 +02:00
|
|
|
use constant MANIFEST_KEY_PROCESS_MAX => 'option-process-max';
|
2019-03-10 11:03:52 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_KEY_PROCESS_MAX);
|
2015-06-14 00:25:49 +02:00
|
|
|
|
|
|
|
# Information about the database that was backed up
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_KEY_DB_ID => 'db-id';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_DB_ID);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_KEY_SYSTEM_ID => 'db-system-id';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_SYSTEM_ID);
|
|
|
|
use constant MANIFEST_KEY_CATALOG => 'db-catalog-version';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_CATALOG);
|
|
|
|
use constant MANIFEST_KEY_CONTROL => 'db-control-version';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_CONTROL);
|
2016-05-11 15:21:39 +02:00
|
|
|
use constant MANIFEST_KEY_DB_LAST_SYSTEM_ID => 'db-last-system-id';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_DB_LAST_SYSTEM_ID);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_KEY_DB_VERSION => 'db-version';
|
|
|
|
push @EXPORT, qw(MANIFEST_KEY_DB_VERSION);
|
|
|
|
|
|
|
|
# Subkeys used for path/file/link info
|
|
|
|
use constant MANIFEST_SUBKEY_CHECKSUM => 'checksum';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM);
|
2016-12-13 01:54:07 +02:00
|
|
|
use constant MANIFEST_SUBKEY_CHECKSUM_PAGE => 'checksum-page';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE);
|
|
|
|
use constant MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR => 'checksum-page-error';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_DESTINATION => 'destination';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_DESTINATION);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SUBKEY_FILE => 'file';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_FILE);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_FUTURE => 'future';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_FUTURE);
|
|
|
|
use constant MANIFEST_SUBKEY_GROUP => 'group';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_GROUP);
|
2016-09-04 15:19:44 +02:00
|
|
|
use constant MANIFEST_SUBKEY_MASTER => 'master';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_MASTER);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_MODE => 'mode';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_MODE);
|
|
|
|
use constant MANIFEST_SUBKEY_TIMESTAMP => 'timestamp';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_TIMESTAMP);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SUBKEY_TYPE => 'type';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_TYPE);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_PATH => 'path';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_PATH);
|
|
|
|
use constant MANIFEST_SUBKEY_REFERENCE => 'reference';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_REFERENCE);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SUBKEY_REPO_SIZE => 'repo-size';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_REPO_SIZE);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_SIZE => 'size';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_SIZE);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant MANIFEST_SUBKEY_TABLESPACE_ID => 'tablespace-id';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_ID);
|
|
|
|
use constant MANIFEST_SUBKEY_TABLESPACE_NAME => 'tablespace-name';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_TABLESPACE_NAME);
|
2015-06-14 00:25:49 +02:00
|
|
|
use constant MANIFEST_SUBKEY_USER => 'user';
|
|
|
|
push @EXPORT, qw(MANIFEST_SUBKEY_USER);
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# Database locations for important files/paths
|
|
|
|
####################################################################################################################################
|
2017-09-01 18:29:34 +02:00
|
|
|
use constant DB_PATH_ARCHIVESTATUS => 'archive_status';
|
|
|
|
push @EXPORT, qw(DB_PATH_ARCHIVESTATUS);
|
2016-08-25 17:25:46 +02:00
|
|
|
use constant DB_PATH_BASE => 'base';
|
|
|
|
push @EXPORT, qw(DB_PATH_BASE);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant DB_PATH_GLOBAL => 'global';
|
|
|
|
push @EXPORT, qw(DB_PATH_GLOBAL);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant DB_PATH_PGDYNSHMEM => 'pg_dynshmem';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGDYNSHMEM);
|
2016-08-25 17:25:46 +02:00
|
|
|
use constant DB_PATH_PGMULTIXACT => 'pg_multixact';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGMULTIXACT);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant DB_PATH_PGNOTIFY => 'pg_notify';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGNOTIFY);
|
|
|
|
use constant DB_PATH_PGREPLSLOT => 'pg_replslot';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGREPLSLOT);
|
|
|
|
use constant DB_PATH_PGSERIAL => 'pg_serial';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGSERIAL);
|
|
|
|
use constant DB_PATH_PGSNAPSHOTS => 'pg_snapshots';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGSNAPSHOTS);
|
|
|
|
use constant DB_PATH_PGSTATTMP => 'pg_stat_tmp';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGSTATTMP);
|
|
|
|
use constant DB_PATH_PGSUBTRANS => 'pg_subtrans';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGSUBTRANS);
|
2016-04-15 04:50:02 +02:00
|
|
|
use constant DB_PATH_PGTBLSPC => 'pg_tblspc';
|
|
|
|
push @EXPORT, qw(DB_PATH_PGTBLSPC);
|
2016-05-14 16:33:12 +02:00
|
|
|
|
|
|
|
use constant DB_FILE_BACKUPLABEL => 'backup_label';
|
|
|
|
push @EXPORT, qw(DB_FILE_BACKUPLABEL);
|
|
|
|
use constant DB_FILE_BACKUPLABELOLD => DB_FILE_BACKUPLABEL . '.old';
|
|
|
|
push @EXPORT, qw(DB_FILE_BACKUPLABELOLD);
|
|
|
|
use constant DB_FILE_PGCONTROL => DB_PATH_GLOBAL . '/pg_control';
|
|
|
|
push @EXPORT, qw(DB_FILE_PGCONTROL);
|
2016-12-13 01:54:07 +02:00
|
|
|
use constant DB_FILE_PGFILENODEMAP => 'pg_filenode.map';
|
|
|
|
push @EXPORT, qw(DB_FILE_PGFILENODEMAP);
|
|
|
|
use constant DB_FILE_PGINTERNALINIT => 'pg_internal.init';
|
|
|
|
push @EXPORT, qw(DB_FILE_PGINTERNALINIT);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant DB_FILE_PGVERSION => 'PG_VERSION';
|
|
|
|
push @EXPORT, qw(DB_FILE_PGVERSION);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant DB_FILE_POSTGRESQLAUTOCONFTMP => 'postgresql.auto.conf.tmp';
|
|
|
|
push @EXPORT, qw(DB_FILE_POSTGRESQLAUTOCONFTMP);
|
2016-07-30 00:44:53 +02:00
|
|
|
use constant DB_FILE_POSTMASTEROPTS => 'postmaster.opts';
|
|
|
|
push @EXPORT, qw(DB_FILE_POSTMASTEROPTS);
|
|
|
|
use constant DB_FILE_POSTMASTERPID => 'postmaster.pid';
|
|
|
|
push @EXPORT, qw(DB_FILE_POSTMASTERPID);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant DB_FILE_RECOVERYCONF => 'recovery.conf';
|
|
|
|
push @EXPORT, qw(DB_FILE_RECOVERYCONF);
|
2019-10-01 19:20:43 +02:00
|
|
|
use constant DB_FILE_RECOVERYSIGNAL => 'recovery.signal';
|
|
|
|
push @EXPORT, qw(DB_FILE_RECOVERYSIGNAL);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant DB_FILE_RECOVERYDONE => 'recovery.done';
|
|
|
|
push @EXPORT, qw(DB_FILE_RECOVERYDONE);
|
2019-10-01 19:20:43 +02:00
|
|
|
use constant DB_FILE_STANDBYSIGNAL => 'standby.signal';
|
|
|
|
push @EXPORT, qw(DB_FILE_STANDBYSIGNAL);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant DB_FILE_TABLESPACEMAP => 'tablespace_map';
|
|
|
|
push @EXPORT, qw(DB_FILE_TABLESPACEMAP);
|
2016-04-15 04:50:02 +02:00
|
|
|
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant DB_FILE_PREFIX_TMP => 'pgsql_tmp';
|
|
|
|
push @EXPORT, qw(DB_FILE_PREFIX_TMP);
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# Manifest locations for important files/paths
|
|
|
|
####################################################################################################################################
|
2016-08-25 17:25:46 +02:00
|
|
|
use constant MANIFEST_PATH_BASE => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_BASE;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_BASE);
|
|
|
|
use constant MANIFEST_PATH_GLOBAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_GLOBAL;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_GLOBAL);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant MANIFEST_PATH_PGDYNSHMEM => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGDYNSHMEM;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGDYNSHMEM);
|
2016-08-25 17:25:46 +02:00
|
|
|
use constant MANIFEST_PATH_PGMULTIXACT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGMULTIXACT;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGMULTIXACT);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant MANIFEST_PATH_PGNOTIFY => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGNOTIFY;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGNOTIFY);
|
|
|
|
use constant MANIFEST_PATH_PGREPLSLOT => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGREPLSLOT;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGREPLSLOT);
|
|
|
|
use constant MANIFEST_PATH_PGSERIAL => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSERIAL;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGSERIAL);
|
|
|
|
use constant MANIFEST_PATH_PGSNAPSHOTS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSNAPSHOTS;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGSNAPSHOTS);
|
|
|
|
use constant MANIFEST_PATH_PGSTATTMP => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSTATTMP;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGSTATTMP);
|
|
|
|
use constant MANIFEST_PATH_PGSUBTRANS => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGSUBTRANS;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGSUBTRANS);
|
2016-08-25 17:25:46 +02:00
|
|
|
use constant MANIFEST_PATH_PGTBLSPC => MANIFEST_TARGET_PGDATA . '/' . DB_PATH_PGTBLSPC;
|
|
|
|
push @EXPORT, qw(MANIFEST_PATH_PGTBLSPC);
|
2016-07-30 00:44:53 +02:00
|
|
|
|
2016-05-16 23:59:26 +02:00
|
|
|
use constant MANIFEST_FILE_BACKUPLABEL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABEL;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_BACKUPLABEL);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant MANIFEST_FILE_BACKUPLABELOLD => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_BACKUPLABELOLD;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_BACKUPLABELOLD);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant MANIFEST_FILE_PGCONTROL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_PGCONTROL;
|
2016-04-15 04:50:02 +02:00
|
|
|
push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant MANIFEST_FILE_POSTGRESQLAUTOCONFTMP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTGRESQLAUTOCONFTMP;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_PGCONTROL);
|
|
|
|
use constant MANIFEST_FILE_POSTMASTEROPTS => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTEROPTS;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_POSTMASTEROPTS);
|
|
|
|
use constant MANIFEST_FILE_POSTMASTERPID => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_POSTMASTERPID;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_POSTMASTERPID);
|
|
|
|
use constant MANIFEST_FILE_RECOVERYCONF => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYCONF;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_RECOVERYCONF);
|
2019-10-01 19:20:43 +02:00
|
|
|
use constant MANIFEST_FILE_RECOVERYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYSIGNAL;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_RECOVERYSIGNAL);
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
use constant MANIFEST_FILE_RECOVERYDONE => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_RECOVERYDONE;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_RECOVERYDONE);
|
2019-10-01 19:20:43 +02:00
|
|
|
use constant MANIFEST_FILE_STANDBYSIGNAL => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_STANDBYSIGNAL;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_STANDBYSIGNAL);
|
2016-05-14 16:33:12 +02:00
|
|
|
use constant MANIFEST_FILE_TABLESPACEMAP => MANIFEST_TARGET_PGDATA . '/' . DB_FILE_TABLESPACEMAP;
|
|
|
|
push @EXPORT, qw(MANIFEST_FILE_TABLESPACEMAP);
|
2016-04-15 04:50:02 +02:00
|
|
|
|
2016-05-11 15:21:39 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# Minimum ID for a user object in postgres
|
|
|
|
####################################################################################################################################
|
|
|
|
use constant DB_USER_OBJECT_MINIMUM_ID => 16384;
|
|
|
|
push @EXPORT, qw(DB_USER_OBJECT_MINIMUM_ID);
|
|
|
|
|
2015-01-07 17:59:43 +02:00
|
|
|
####################################################################################################################################
|
2015-06-14 00:25:49 +02:00
|
|
|
# new
|
2015-01-07 17:59:43 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
sub new
|
|
|
|
{
|
2015-08-29 20:20:46 +02:00
|
|
|
my $class = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my
|
|
|
|
(
|
|
|
|
$strOperation,
|
2017-06-09 23:51:41 +02:00
|
|
|
$strFileName,
|
|
|
|
$bLoad,
|
|
|
|
$oStorage,
|
2017-09-01 18:29:34 +02:00
|
|
|
$strDbVersion,
|
2018-07-16 23:25:15 +02:00
|
|
|
$iDbCatalogVersion,
|
2017-11-06 19:51:12 +02:00
|
|
|
$strCipherPass, # Passphrase to open the manifest if encrypted
|
|
|
|
$strCipherPassSub, # Passphrase to encrypt the backup files
|
2015-08-29 20:20:46 +02:00
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
2016-08-11 23:32:28 +02:00
|
|
|
__PACKAGE__ . '->new', \@_,
|
2015-08-29 20:20:46 +02:00
|
|
|
{name => 'strFileName', trace => true},
|
2017-09-01 18:29:34 +02:00
|
|
|
{name => 'bLoad', optional => true, default => true, trace => true},
|
2017-06-09 23:51:41 +02:00
|
|
|
{name => 'oStorage', optional => true, default => storageRepo(), trace => true},
|
2017-09-01 18:29:34 +02:00
|
|
|
{name => 'strDbVersion', optional => true, trace => true},
|
2018-07-16 23:25:15 +02:00
|
|
|
{name => 'iDbCatalogVersion', optional => true, trace => true},
|
2017-11-06 19:51:12 +02:00
|
|
|
{name => 'strCipherPass', optional => true, redact => true},
|
|
|
|
{name => 'strCipherPassSub', optional => true, redact => true},
|
2015-08-29 20:20:46 +02:00
|
|
|
);
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2015-06-14 00:25:49 +02:00
|
|
|
# Init object and store variables
|
2020-03-09 23:41:59 +02:00
|
|
|
my $self = $class->SUPER::new(
|
|
|
|
$oStorage, $strFileName, {bLoad => $bLoad, strCipherPass => $strCipherPass, strCipherPassSub => $strCipherPassSub});
|
2015-01-20 21:13:35 +02:00
|
|
|
|
2018-07-16 23:25:15 +02:00
|
|
|
# If manifest not loaded from a file then the db version and catalog version must be set
|
2017-09-01 18:29:34 +02:00
|
|
|
if (!$bLoad)
|
|
|
|
{
|
2018-07-16 23:25:15 +02:00
|
|
|
if (!(defined($strDbVersion) && defined($iDbCatalogVersion)))
|
2017-09-01 18:29:34 +02:00
|
|
|
{
|
2018-07-16 23:25:15 +02:00
|
|
|
confess &log(ASSERT, 'strDbVersion and iDbCatalogVersion must be provided with bLoad = false');
|
2017-09-01 18:29:34 +02:00
|
|
|
}
|
|
|
|
|
2019-07-05 23:25:01 +02:00
|
|
|
# Force the version to a string since newer versions of JSON::PP lose track of the fact that it is one
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION, undef, $strDbVersion . '');
|
2018-07-16 23:25:15 +02:00
|
|
|
$self->numericSet(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG, undef, $iDbCatalogVersion);
|
2017-09-01 18:29:34 +02:00
|
|
|
}
|
|
|
|
|
2018-08-28 22:22:30 +02:00
|
|
|
# Mark the manifest as built if it was loaded from a file
|
|
|
|
$self->{bBuilt} = $bLoad;
|
|
|
|
|
2015-08-29 20:20:46 +02:00
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'self', value => $self}
|
|
|
|
);
|
2015-01-07 17:59:43 +02:00
|
|
|
}
|
|
|
|
|
2015-01-20 21:13:35 +02:00
|
|
|
####################################################################################################################################
|
2015-06-14 00:25:49 +02:00
|
|
|
# save
|
2015-01-20 21:13:35 +02:00
|
|
|
#
|
2015-01-20 21:42:22 +02:00
|
|
|
# Save the manifest.
|
2015-01-20 21:13:35 +02:00
|
|
|
####################################################################################################################################
|
2015-01-20 21:42:22 +02:00
|
|
|
sub save
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
2015-08-29 20:20:46 +02:00
|
|
|
# Assign function parameters, defaults, and log debug info
|
2016-08-11 23:32:28 +02:00
|
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . '->save');
|
2015-08-29 20:20:46 +02:00
|
|
|
|
2015-06-14 00:25:49 +02:00
|
|
|
# Call inherited save
|
|
|
|
$self->SUPER::save();
|
2015-08-29 20:20:46 +02:00
|
|
|
|
|
|
|
# Return from function and log return values if any
|
2016-08-11 23:32:28 +02:00
|
|
|
return logDebugReturn($strOperation);
|
2015-01-20 21:42:22 +02:00
|
|
|
}
|
2015-01-20 21:13:35 +02:00
|
|
|
|
2015-01-07 17:59:43 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
# get
|
2015-01-08 19:04:56 +02:00
|
|
|
#
|
2016-04-15 04:50:02 +02:00
|
|
|
# Get a value.
|
2015-01-08 19:04:56 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
sub get
|
2015-01-08 19:04:56 +02:00
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strSection = shift;
|
|
|
|
my $strKey = shift;
|
|
|
|
my $strSubKey = shift;
|
2016-04-15 04:50:02 +02:00
|
|
|
my $bRequired = shift;
|
|
|
|
my $oDefault = shift;
|
2015-01-08 19:04:56 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
my $oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, false);
|
2015-01-08 19:04:56 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
if (!defined($oValue) && defined($strKey) && defined($strSubKey) &&
|
|
|
|
($strSection eq MANIFEST_SECTION_TARGET_FILE || $strSection eq MANIFEST_SECTION_TARGET_PATH ||
|
|
|
|
$strSection eq MANIFEST_SECTION_TARGET_LINK) &&
|
|
|
|
($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP ||
|
2016-09-04 15:19:44 +02:00
|
|
|
$strSubKey eq MANIFEST_SUBKEY_MODE || $strSubKey eq MANIFEST_SUBKEY_MASTER) &&
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->test($strSection, $strKey))
|
|
|
|
{
|
|
|
|
$oValue = $self->SUPER::get("${strSection}:default", $strSubKey, undef, $bRequired, $oDefault);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$oValue = $self->SUPER::get($strSection, $strKey, $strSubKey, $bRequired, $oDefault);
|
|
|
|
}
|
|
|
|
|
|
|
|
return $oValue;
|
2015-01-08 19:04:56 +02:00
|
|
|
}
|
|
|
|
|
2015-01-20 23:05:15 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
# boolGet
|
2015-01-20 23:05:15 +02:00
|
|
|
#
|
2016-04-15 04:50:02 +02:00
|
|
|
# Get a numeric value.
|
2015-01-20 23:05:15 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
sub boolGet
|
2015-01-20 23:05:15 +02:00
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strSection = shift;
|
|
|
|
my $strValue = shift;
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strSubValue = shift;
|
|
|
|
my $bRequired = shift;
|
|
|
|
my $bDefault = shift;
|
2015-01-20 23:05:15 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
return $self->get($strSection, $strValue, $strSubValue, $bRequired,
|
|
|
|
defined($bDefault) ? ($bDefault ? INI_TRUE : INI_FALSE) : undef) ? true : false;
|
2015-01-20 23:05:15 +02:00
|
|
|
}
|
|
|
|
|
2015-01-08 19:04:56 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
# numericGet
|
2015-01-08 19:04:56 +02:00
|
|
|
#
|
2016-04-15 04:50:02 +02:00
|
|
|
# Get a numeric value.
|
2015-01-08 19:04:56 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
sub numericGet
|
2015-01-08 19:04:56 +02:00
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strSection = shift;
|
2015-01-21 20:57:58 +02:00
|
|
|
my $strValue = shift;
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strSubValue = shift;
|
|
|
|
my $bRequired = shift;
|
|
|
|
my $nDefault = shift;
|
2015-01-08 19:04:56 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
return $self->get($strSection, $strValue, $strSubValue, $bRequired,
|
|
|
|
defined($nDefault) ? $nDefault + 0 : undef) + 0;
|
|
|
|
}
|
2015-01-08 19:04:56 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# tablespacePathGet
|
|
|
|
#
|
|
|
|
# Get the unique path assigned by Postgres for the tablespace.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub tablespacePathGet
|
|
|
|
{
|
|
|
|
my $self = shift;
|
2015-01-21 20:57:58 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
return('PG_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) .
|
|
|
|
'_' . $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_CATALOG));
|
|
|
|
}
|
2015-01-21 20:57:58 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# dbPathGet
|
|
|
|
#
|
|
|
|
# Convert a repo path to where the file actually belongs in the db.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub dbPathGet
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strDbPath = shift;
|
|
|
|
my $strFile = shift;
|
2015-01-21 02:04:45 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strDbFile = defined($strDbPath) ? "${strDbPath}/" : '';
|
2015-01-23 03:11:33 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
if (index($strFile, MANIFEST_TARGET_PGDATA . '/') == 0)
|
|
|
|
{
|
|
|
|
$strDbFile .= substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1);
|
2015-01-21 02:04:45 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
else
|
2015-06-14 00:25:49 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$strDbFile .= $strFile;
|
2015-06-14 00:25:49 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
return $strDbFile;
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# repoPathGet
|
|
|
|
#
|
|
|
|
# Convert a database path to where to file is located in the repo.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub repoPathGet
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strTarget = shift;
|
|
|
|
my $strFile = shift;
|
|
|
|
|
|
|
|
my $strRepoFile = $strTarget;
|
|
|
|
|
2018-01-30 23:13:54 +02:00
|
|
|
if ($self->isTargetTablespace($strTarget) &&
|
|
|
|
($self->dbVersion() >= PG_VERSION_90))
|
2015-01-20 23:05:15 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$strRepoFile .= '/' . $self->tablespacePathGet();
|
2015-01-20 23:05:15 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
if (defined($strFile))
|
2015-01-21 01:00:03 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$strRepoFile .= "/${strFile}";
|
2015-01-21 01:00:03 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
return $strRepoFile;
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# isTargetValid
|
|
|
|
#
|
|
|
|
# Determine if a target is valid.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub isTargetValid
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strTarget = shift;
|
|
|
|
my $bError = shift;
|
|
|
|
|
|
|
|
if (!defined($strTarget))
|
2015-01-23 03:11:33 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
confess &log(ASSERT, 'target is not defined');
|
2015-01-23 03:11:33 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
if (!$self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget))
|
2015-01-08 19:04:56 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
if (defined($bError) && $bError)
|
2015-01-08 19:04:56 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
confess &log(ASSERT, "${strTarget} is not a valid target");
|
2015-01-08 19:04:56 +02:00
|
|
|
}
|
2015-01-07 17:59:43 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
return false;
|
2015-01-07 17:59:43 +02:00
|
|
|
}
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
return true;
|
2015-01-07 17:59:43 +02:00
|
|
|
}
|
|
|
|
|
2016-02-02 21:33:15 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
# isTargetLink
|
2016-02-02 21:33:15 +02:00
|
|
|
#
|
2016-04-15 04:50:02 +02:00
|
|
|
# Determine if a target is a link.
|
2016-02-02 21:33:15 +02:00
|
|
|
####################################################################################################################################
|
2016-04-15 04:50:02 +02:00
|
|
|
sub isTargetLink
|
2016-02-02 21:33:15 +02:00
|
|
|
{
|
|
|
|
my $self = shift;
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strTarget = shift;
|
2016-02-02 21:33:15 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->isTargetValid($strTarget, true);
|
|
|
|
|
|
|
|
return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TYPE, MANIFEST_VALUE_LINK);
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# isTargetFile
|
|
|
|
#
|
|
|
|
# Determine if a target is a file link.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub isTargetFile
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strTarget = shift;
|
|
|
|
|
|
|
|
$self->isTargetValid($strTarget, true);
|
|
|
|
|
|
|
|
return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_FILE);
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# isTargetTablespace
|
|
|
|
#
|
|
|
|
# Determine if a target is a tablespace.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub isTargetTablespace
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strTarget = shift;
|
|
|
|
|
|
|
|
$self->isTargetValid($strTarget, true);
|
|
|
|
|
|
|
|
return $self->test(MANIFEST_SECTION_BACKUP_TARGET, $strTarget, MANIFEST_SUBKEY_TABLESPACE_ID);
|
2016-02-02 21:33:15 +02:00
|
|
|
}
|
|
|
|
|
2018-11-01 17:31:25 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# checkDelta
|
|
|
|
#
|
|
|
|
# Determine if the delta option should be enabled. Only called if delta has not yet been enabled.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub checkDelta
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
$strLastBackupSource,
|
|
|
|
$bOnlineSame,
|
|
|
|
$strTimelineCurrent,
|
|
|
|
$strTimelineLast,
|
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
|
|
|
__PACKAGE__ . '->checkDelta', \@_,
|
|
|
|
{name => 'strLastBackupSource'},
|
|
|
|
{name => 'bOnlineSame'},
|
|
|
|
{name => 'strTimelineCurrent', required => false},
|
|
|
|
{name => 'strTimelineLast', required => false},
|
|
|
|
);
|
|
|
|
|
|
|
|
my $bDelta = false;
|
|
|
|
|
|
|
|
# Determine if a timeline switch has occurred
|
|
|
|
if (defined($strTimelineLast) && defined($strTimelineCurrent))
|
|
|
|
{
|
|
|
|
# If there is a prior backup, check if a timeline switch has occurred since then
|
|
|
|
if ($strTimelineLast ne $strTimelineCurrent)
|
|
|
|
{
|
|
|
|
&log(WARN, "a timeline switch has occurred since the ${strLastBackupSource} backup, enabling delta checksum");
|
|
|
|
$bDelta = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# If delta was not set above and there is a change in the online option, then set delta option
|
|
|
|
if (!$bDelta && !$bOnlineSame)
|
|
|
|
{
|
|
|
|
&log(WARN, "the online option has changed since the ${strLastBackupSource} backup, enabling delta checksum");
|
|
|
|
$bDelta = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'bDelta', value => $bDelta, trace => true},
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# checkDeltaFile
|
|
|
|
#
|
|
|
|
# Determine if the delta option should be enabled. Only called if delta has not yet been enabled.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub checkDeltaFile
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
$stryFileList,
|
|
|
|
$oPriorManifest,
|
|
|
|
$lTimeBegin,
|
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
|
|
|
__PACKAGE__ . '->checkDeltaFile', \@_,
|
|
|
|
{name => 'stryFileList'},
|
|
|
|
{name => 'oPriorManifest', required => false},
|
|
|
|
{name => 'lTimeBegin', required => false},
|
|
|
|
);
|
|
|
|
|
|
|
|
my $bDelta = false;
|
|
|
|
|
|
|
|
# Loop though all files
|
|
|
|
foreach my $strName (@{$stryFileList})
|
|
|
|
{
|
|
|
|
# If $lTimeBegin is defined, then this is not an aborted manifest so check if modification time is in the future (in this
|
|
|
|
# backup OR the last backup) then enable delta and exit
|
|
|
|
if (defined($lTimeBegin) &&
|
|
|
|
($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin ||
|
|
|
|
(defined($oPriorManifest) &&
|
|
|
|
$oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y'))))
|
|
|
|
{
|
|
|
|
&log(WARN, "file $strName has timestamp in the future, enabling delta checksum");
|
|
|
|
$bDelta = true;
|
|
|
|
last;
|
|
|
|
}
|
|
|
|
|
|
|
|
# If the time on the file is earlier than the last manifest time or the size is different but the timestamp is the
|
|
|
|
# same, then enable delta and exit
|
|
|
|
if (defined($oPriorManifest) && $oPriorManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) &&
|
|
|
|
($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) <
|
|
|
|
$oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ||
|
|
|
|
($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) !=
|
|
|
|
$oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) &&
|
|
|
|
$self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ==
|
|
|
|
$oPriorManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP))))
|
|
|
|
{
|
|
|
|
&log(WARN, "file $strName timestamp in the past or size changed but timestamp did not, enabling delta checksum");
|
|
|
|
$bDelta = true;
|
|
|
|
last;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'bDelta', value => $bDelta, trace => true},
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2015-02-01 01:15:08 +02:00
|
|
|
####################################################################################################################################
|
2015-06-14 00:25:49 +02:00
|
|
|
# build
|
2015-02-01 01:23:37 +02:00
|
|
|
#
|
|
|
|
# Build the manifest object.
|
2015-02-01 01:15:08 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
sub build
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
2015-08-29 20:20:46 +02:00
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my
|
|
|
|
(
|
|
|
|
$strOperation,
|
2017-06-09 23:51:41 +02:00
|
|
|
$oStorageDbMaster,
|
2016-04-15 04:50:02 +02:00
|
|
|
$strPath,
|
2015-08-29 20:20:46 +02:00
|
|
|
$oLastManifest,
|
2016-02-20 02:32:02 +02:00
|
|
|
$bOnline,
|
2018-09-19 17:12:45 +02:00
|
|
|
$bDelta,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hTablespaceMap,
|
|
|
|
$hDatabaseMap,
|
2018-08-27 21:51:05 +02:00
|
|
|
$rhExclude,
|
2018-11-01 17:31:25 +02:00
|
|
|
$strTimelineCurrent,
|
|
|
|
$strTimelineLast,
|
2016-04-15 04:50:02 +02:00
|
|
|
$strLevel,
|
|
|
|
$bTablespace,
|
|
|
|
$strParentPath,
|
2018-08-27 21:51:05 +02:00
|
|
|
$strFilter,
|
2018-08-28 22:27:10 +02:00
|
|
|
$iLevel,
|
2015-08-29 20:20:46 +02:00
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
2016-08-11 23:32:28 +02:00
|
|
|
__PACKAGE__ . '->build', \@_,
|
2017-06-09 23:51:41 +02:00
|
|
|
{name => 'oStorageDbMaster'},
|
2016-04-15 04:50:02 +02:00
|
|
|
{name => 'strPath'},
|
2015-08-29 20:20:46 +02:00
|
|
|
{name => 'oLastManifest', required => false},
|
2016-02-20 02:32:02 +02:00
|
|
|
{name => 'bOnline'},
|
2018-09-19 17:12:45 +02:00
|
|
|
{name => 'bDelta'},
|
2016-11-30 21:36:39 +02:00
|
|
|
{name => 'hTablespaceMap', required => false},
|
|
|
|
{name => 'hDatabaseMap', required => false},
|
2018-08-27 21:51:05 +02:00
|
|
|
{name => 'rhExclude', required => false},
|
2018-11-01 17:31:25 +02:00
|
|
|
{name => 'strTimelineCurrent', required => false},
|
|
|
|
{name => 'strTimelineLast', required => false},
|
2016-04-15 04:50:02 +02:00
|
|
|
{name => 'strLevel', required => false},
|
|
|
|
{name => 'bTablespace', required => false},
|
|
|
|
{name => 'strParentPath', required => false},
|
2018-08-27 21:51:05 +02:00
|
|
|
{name => 'strFilter', required => false},
|
2018-08-28 22:27:10 +02:00
|
|
|
{name => 'iLevel', required => false, default => 0},
|
2015-08-29 20:20:46 +02:00
|
|
|
);
|
2015-02-03 01:33:11 +02:00
|
|
|
|
2018-08-28 22:27:10 +02:00
|
|
|
# Limit recursion to something reasonable (if more then we are very likely in a link loop)
|
|
|
|
if ($iLevel >= 16)
|
|
|
|
{
|
|
|
|
confess &log(
|
|
|
|
ERROR,
|
|
|
|
"recursion in manifest build exceeds depth of ${iLevel}: ${strLevel}\n" .
|
|
|
|
'HINT: is there a link loop in $PGDATA?',
|
|
|
|
ERROR_FORMAT);
|
|
|
|
}
|
|
|
|
|
2015-02-01 01:15:08 +02:00
|
|
|
if (!defined($strLevel))
|
|
|
|
{
|
2018-08-28 22:22:30 +02:00
|
|
|
# Don't allow the manifest to be built more than once
|
|
|
|
if ($self->{bBuilt})
|
|
|
|
{
|
|
|
|
confess &log(ASSERT, "manifest has already been built");
|
|
|
|
}
|
|
|
|
|
|
|
|
$self->{bBuilt} = true;
|
|
|
|
|
|
|
|
# Set initial level
|
2016-04-15 04:50:02 +02:00
|
|
|
$strLevel = MANIFEST_TARGET_PGDATA;
|
2015-02-03 01:33:11 +02:00
|
|
|
|
2016-02-20 02:32:02 +02:00
|
|
|
# If not online then build the tablespace map from pg_tblspc path
|
2016-11-30 21:36:39 +02:00
|
|
|
if (!$bOnline && !defined($hTablespaceMap))
|
2015-02-01 01:23:37 +02:00
|
|
|
{
|
2017-06-09 23:51:41 +02:00
|
|
|
my $hTablespaceManifest = $oStorageDbMaster->manifest($strPath . '/' . DB_PATH_PGTBLSPC);
|
2016-11-30 21:36:39 +02:00
|
|
|
$hTablespaceMap = {};
|
2015-02-01 01:23:37 +02:00
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
foreach my $strOid (sort(CORE::keys(%{$hTablespaceManifest})))
|
2015-02-01 01:23:37 +02:00
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
if ($strOid eq '.' or $strOid eq '..')
|
2015-02-01 01:23:37 +02:00
|
|
|
{
|
|
|
|
next;
|
|
|
|
}
|
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
logDebugMisc($strOperation, "found tablespace ${strOid} in offline mode");
|
2015-02-01 01:23:37 +02:00
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
$hTablespaceMap->{$strOid} = "ts${strOid}";
|
2015-02-01 01:23:37 +02:00
|
|
|
}
|
|
|
|
}
|
2018-11-01 17:31:25 +02:00
|
|
|
|
|
|
|
# If there is a last manifest, then check to see if delta checksum should be enabled
|
|
|
|
if (defined($oLastManifest) && !$bDelta)
|
|
|
|
{
|
|
|
|
$bDelta = $self->checkDelta(
|
|
|
|
'last', $oLastManifest->boolTest(MANIFEST_SECTION_BACKUP_OPTION, MANIFEST_KEY_ONLINE, undef, $bOnline),
|
|
|
|
$strTimelineCurrent, $strTimelineLast);
|
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH, $strPath);
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TYPE,
|
|
|
|
$strLevel eq MANIFEST_TARGET_PGDATA ? MANIFEST_VALUE_PATH : MANIFEST_VALUE_LINK);
|
|
|
|
|
|
|
|
if ($bTablespace)
|
|
|
|
{
|
|
|
|
my $iTablespaceId = (split('\/', $strLevel))[1];
|
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
if (!defined($hTablespaceMap->{$iTablespaceId}))
|
2016-04-15 04:50:02 +02:00
|
|
|
{
|
|
|
|
confess &log(ASSERT, "tablespace with oid ${iTablespaceId} not found in tablespace map\n" .
|
|
|
|
"HINT: was a tablespace created or dropped during the backup?");
|
|
|
|
}
|
|
|
|
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_ID, $iTablespaceId);
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_TABLESPACE_NAME,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hTablespaceMap->{$iTablespaceId});
|
2016-04-15 04:50:02 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
if (index($strPath, '/') != 0)
|
2016-02-02 21:33:15 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
if (!defined($strParentPath))
|
|
|
|
{
|
|
|
|
confess &log(ASSERT, "cannot get manifest for '${strPath}' when no parent path is specified");
|
|
|
|
}
|
|
|
|
|
2017-06-09 23:51:41 +02:00
|
|
|
$strPath = $oStorageDbMaster->pathAbsolute($strParentPath, $strPath);
|
2016-02-02 21:33:15 +02:00
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
|
|
|
|
# Get the manifest for this level
|
2018-08-31 22:06:40 +02:00
|
|
|
my $hManifest = $oStorageDbMaster->manifest($strPath, {strFilter => $strFilter});
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strManifestType = MANIFEST_VALUE_LINK;
|
2015-02-01 01:15:08 +02:00
|
|
|
|
|
|
|
# Loop though all paths/files/links in the manifest
|
2016-11-30 21:36:39 +02:00
|
|
|
foreach my $strName (sort(CORE::keys(%{$hManifest})))
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strFile = $strLevel;
|
|
|
|
|
|
|
|
if ($strName ne '.')
|
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
if ($strManifestType eq MANIFEST_VALUE_LINK && $hManifest->{$strName}{type} eq 'l')
|
2016-08-15 23:11:45 +02:00
|
|
|
{
|
2016-08-15 23:23:37 +02:00
|
|
|
confess &log(ERROR, 'link \'' .
|
2016-08-15 23:11:45 +02:00
|
|
|
$self->dbPathGet(
|
|
|
|
$self->get(MANIFEST_SECTION_BACKUP_TARGET, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_PATH), $strLevel) .
|
2016-08-15 23:23:37 +02:00
|
|
|
'\' -> \'' . $self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH) .
|
|
|
|
'\' cannot reference another link', ERROR_LINK_DESTINATION);
|
2016-08-15 23:11:45 +02:00
|
|
|
}
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
if ($strManifestType eq MANIFEST_VALUE_LINK)
|
|
|
|
{
|
|
|
|
$strFile = dirname($strFile);
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH,
|
|
|
|
dirname($self->get(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_PATH)));
|
|
|
|
$self->set(MANIFEST_SECTION_BACKUP_TARGET, $strLevel, MANIFEST_SUBKEY_FILE, $strName);
|
|
|
|
}
|
|
|
|
|
|
|
|
$strFile .= "/${strName}";
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$strManifestType = MANIFEST_VALUE_PATH;
|
|
|
|
}
|
|
|
|
|
2017-09-01 18:29:34 +02:00
|
|
|
# Skip wal directory when doing an online backup. WAL will be restored from the archive or stored in the wal directory at
|
|
|
|
# the end of the backup if the archive-copy option is set.
|
|
|
|
next if ($bOnline && $strFile =~ (qw{^} . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . '\/') &&
|
|
|
|
$strFile !~ ('^' . MANIFEST_TARGET_PGDATA . qw{/} . $self->walPath() . qw{/} . DB_PATH_ARCHIVESTATUS . '$'));
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
|
|
|
|
# Skip all directories and files that start with pgsql_tmp. The files are removed when the server is restarted and the
|
2017-09-04 14:26:57 +02:00
|
|
|
# directories are recreated.
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
next if $strName =~ ('(^|\/)' . DB_FILE_PREFIX_TMP);
|
|
|
|
|
2017-09-04 14:26:57 +02:00
|
|
|
# Skip pg_dynshmem/* since these files cannot be reused on recovery
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGDYNSHMEM . '\/') && $self->dbVersion() >= PG_VERSION_94;
|
|
|
|
|
|
|
|
# Skip pg_notify/* since these files cannot be reused on recovery
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGNOTIFY . '\/') && $self->dbVersion() >= PG_VERSION_90;
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
|
2016-09-04 15:13:13 +02:00
|
|
|
# Skip pg_replslot/* since these files are generally not useful after a restore
|
2017-09-01 18:29:34 +02:00
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGREPLSLOT . '\/') && $self->dbVersion() >= PG_VERSION_94;
|
2016-09-04 15:13:13 +02:00
|
|
|
|
2017-09-04 14:26:57 +02:00
|
|
|
# Skip pg_serial/* since these files are reset
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGSERIAL . '\/') && $self->dbVersion() >= PG_VERSION_91;
|
|
|
|
|
|
|
|
# Skip pg_snapshots/* since these files cannot be reused on recovery
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGSNAPSHOTS . '\/') && $self->dbVersion() >= PG_VERSION_92;
|
|
|
|
|
|
|
|
# Skip temporary statistics in pg_stat_tmp even when stats_temp_directory is set because PGSS_TEXT_FILE is always created
|
|
|
|
# there.
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGSTATTMP . '\/') && $self->dbVersion() >= PG_VERSION_84;
|
|
|
|
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
# Skip pg_subtrans/* since these files are reset
|
|
|
|
next if $strFile =~ ('^' . MANIFEST_PATH_PGSUBTRANS . '\/');
|
|
|
|
|
2017-09-04 14:26:57 +02:00
|
|
|
# Skip pg_internal.init since it is recreated on startup
|
|
|
|
next if $strFile =~ (DB_FILE_PGINTERNALINIT . '$');
|
|
|
|
|
2019-10-01 19:20:43 +02:00
|
|
|
# Skip recovery files
|
|
|
|
if ($self->dbVersion() >= PG_VERSION_12)
|
|
|
|
{
|
|
|
|
next if ($strFile eq MANIFEST_FILE_RECOVERYSIGNAL || $strFile eq MANIFEST_FILE_STANDBYSIGNAL);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
next if ($strFile eq MANIFEST_FILE_RECOVERYDONE || $strFile eq MANIFEST_FILE_RECOVERYCONF);
|
|
|
|
}
|
|
|
|
|
Exclude directories during backup that are cleaned, recreated, or zeroed by PostgreSQL at startup.
These include (depending on the version where they were introduced): pgsql_tmp, pg_dynshmem, pg_notify, pg_replslot, pg_serial, pg_snapshots, pg_stat_tmp, pg_subtrans. The postgresql.auto.conf.tmp file is now excluded in addition to files that were already excluded: backup_label.old, postmaster.opts, postmaster.pid, recovery.conf, recovery.done.
2016-08-16 15:35:16 +02:00
|
|
|
# Skip ignored files
|
|
|
|
if ($strFile eq MANIFEST_FILE_POSTGRESQLAUTOCONFTMP || # postgresql.auto.conf.tmp - temp file for safe writes
|
|
|
|
$strFile eq MANIFEST_FILE_BACKUPLABELOLD || # backup_label.old - old backup labels are not useful
|
|
|
|
$strFile eq MANIFEST_FILE_POSTMASTEROPTS || # postmaster.opts - not useful for backup
|
2019-10-01 19:20:43 +02:00
|
|
|
$strFile eq MANIFEST_FILE_POSTMASTERPID) # postmaster.pid - to avoid confusing postgres after restore
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
|
|
|
next;
|
|
|
|
}
|
|
|
|
|
2018-07-31 00:53:34 +02:00
|
|
|
# If version is greater than 9.0, check for files to exclude
|
|
|
|
if ($self->dbVersion() >= PG_VERSION_90 && $hManifest->{$strName}{type} eq 'f')
|
|
|
|
{
|
2019-08-26 18:05:36 +02:00
|
|
|
# Get the directory name from the manifest; it will be used later to search for existence in the keys
|
2018-07-31 00:53:34 +02:00
|
|
|
my $strDir = dirname($strName);
|
|
|
|
|
|
|
|
# If it is a database data directory (base or tablespace) then check for files to skip
|
|
|
|
if ($strDir =~ '^base\/[0-9]+$' ||
|
|
|
|
$strDir =~ ('^' . $self->tablespacePathGet() . '\/[0-9]+$'))
|
|
|
|
{
|
|
|
|
# Get just the filename
|
|
|
|
my $strBaseName = basename($strName);
|
|
|
|
|
|
|
|
# Skip temp tables (lower t followed by numbers underscore numbers and a dot (segment) or underscore (fork) and/or
|
|
|
|
# segment, e.g. t1234_123, t1234_123.1, t1234_123_vm, t1234_123_fsm.1
|
|
|
|
if ($strBaseName =~ '^t[0-9]+\_[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$')
|
|
|
|
{
|
|
|
|
next;
|
|
|
|
}
|
|
|
|
|
|
|
|
# If version is greater than 9.1 then check for unlogged tables to skip
|
|
|
|
if ($self->dbVersion() >= PG_VERSION_91)
|
|
|
|
{
|
|
|
|
# Exclude all forks for unlogged tables except the init fork (numbers underscore init and optional dot segment)
|
|
|
|
if ($strBaseName =~ '^[0-9]+(|\_(fsm|vm)){0,1}(\.[0-9]+){0,1}$')
|
|
|
|
{
|
|
|
|
# Get the filenode (OID)
|
|
|
|
my ($strFileNode) = $strBaseName =~ '^(\d+)';
|
|
|
|
|
|
|
|
# Add _init to the OID to see if this is an unlogged object
|
|
|
|
$strFileNode = $strDir. "/" . $strFileNode . "_init";
|
|
|
|
|
|
|
|
# If key exists in manifest then skip
|
|
|
|
if (exists($hManifest->{$strFileNode}) && $hManifest->{$strFileNode}{type} eq 'f')
|
|
|
|
{
|
|
|
|
next;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-23 13:47:54 +02:00
|
|
|
# Exclude files requested by the user
|
|
|
|
if (defined($rhExclude))
|
|
|
|
{
|
|
|
|
# Exclusions are based on the name of the file relative to PGDATA
|
|
|
|
my $strPgFile = $self->dbPathGet(undef, $strFile);
|
|
|
|
my $bExclude = false;
|
|
|
|
|
|
|
|
# Iterate through exclusions
|
|
|
|
foreach my $strExclude (sort(keys(%{$rhExclude})))
|
|
|
|
{
|
|
|
|
# If the exclusion ends in / then we must do a prefix match
|
|
|
|
if ($strExclude =~ /\/$/)
|
|
|
|
{
|
|
|
|
if (index($strPgFile, $strExclude) == 0)
|
|
|
|
{
|
|
|
|
$bExclude = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
# Else an exact match or a prefix match with / appended is required
|
|
|
|
elsif ($strPgFile eq $strExclude || index($strPgFile, "${strExclude}/") == 0)
|
|
|
|
{
|
|
|
|
$bExclude = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
# Log everything that gets excluded at a high level so it will hopefully be seen if wrong
|
|
|
|
if ($bExclude)
|
|
|
|
{
|
|
|
|
&log(INFO, "exclude ${strPgFile} from backup using '${strExclude}' exclusion");
|
|
|
|
last;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Skip the file if it was excluded
|
|
|
|
next if $bExclude;
|
|
|
|
}
|
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
my $cType = $hManifest->{$strName}{type};
|
2016-04-15 04:50:02 +02:00
|
|
|
my $strSection = MANIFEST_SECTION_TARGET_PATH;
|
2015-02-01 01:15:08 +02:00
|
|
|
|
|
|
|
if ($cType eq 'f')
|
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$strSection = MANIFEST_SECTION_TARGET_FILE;
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
elsif ($cType eq 'l')
|
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$strSection = MANIFEST_SECTION_TARGET_LINK;
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
elsif ($cType ne 'd')
|
|
|
|
{
|
2019-08-23 13:47:54 +02:00
|
|
|
&log(WARN, "exclude special file '" . $self->dbPathGet(undef, $strFile) . "' from backup");
|
|
|
|
next;
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
# Make sure that DB_PATH_PGTBLSPC contains only absolute links that do not point inside PGDATA
|
|
|
|
my $bTablespace = false;
|
|
|
|
|
|
|
|
if (index($strName, DB_PATH_PGTBLSPC . '/') == 0 && $strLevel eq MANIFEST_TARGET_PGDATA)
|
2016-01-15 14:07:00 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$bTablespace = true;
|
|
|
|
$strFile = MANIFEST_TARGET_PGDATA . '/' . $strName;
|
2016-01-15 14:07:00 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
# Check for files in DB_PATH_PGTBLSPC that are not links
|
2016-11-30 21:36:39 +02:00
|
|
|
if ($hManifest->{$strName}{type} ne 'l')
|
2016-01-15 14:07:00 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
confess &log(ERROR, "${strName} is not a symlink - " . DB_PATH_PGTBLSPC . ' should contain only symlinks',
|
|
|
|
ERROR_LINK_EXPECTED);
|
2016-01-15 14:07:00 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
# Check for tablespaces in PGDATA
|
2016-11-30 21:36:39 +02:00
|
|
|
if (index($hManifest->{$strName}{link_destination}, "${strPath}/") == 0 ||
|
|
|
|
(index($hManifest->{$strName}{link_destination}, '/') != 0 &&
|
2017-06-09 23:51:41 +02:00
|
|
|
index($oStorageDbMaster->pathAbsolute($strPath . '/' . DB_PATH_PGTBLSPC,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hManifest->{$strName}{link_destination}) . '/', "${strPath}/") == 0))
|
2016-01-15 14:07:00 +02:00
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
confess &log(ERROR, 'tablespace symlink ' . $hManifest->{$strName}{link_destination} .
|
2019-12-02 17:49:25 +02:00
|
|
|
' destination must not be in $PGDATA', ERROR_LINK_DESTINATION);
|
2016-01-15 14:07:00 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-02-01 01:15:08 +02:00
|
|
|
# User and group required for all types
|
2016-11-30 21:36:39 +02:00
|
|
|
if (defined($hManifest->{$strName}{user}))
|
2016-06-27 03:01:20 +02:00
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_USER, $hManifest->{$strName}{user});
|
2016-06-27 03:01:20 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_USER, false);
|
|
|
|
}
|
|
|
|
|
2016-11-30 21:36:39 +02:00
|
|
|
if (defined($hManifest->{$strName}{group}))
|
2016-06-27 03:01:20 +02:00
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_GROUP, $hManifest->{$strName}{group});
|
2016-06-27 03:01:20 +02:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_GROUP, false);
|
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
|
|
|
|
# Mode for required file and path type only
|
|
|
|
if ($cType eq 'f' || $cType eq 'd')
|
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_MODE, $hManifest->{$strName}{mode});
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
# Modification time and size required for file type only
|
|
|
|
if ($cType eq 'f')
|
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_TIMESTAMP,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hManifest->{$strName}{modification_time} + 0);
|
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_SIZE, $hManifest->{$strName}{size} + 0);
|
2016-09-04 15:19:44 +02:00
|
|
|
$self->boolSet($strSection, $strFile, MANIFEST_SUBKEY_MASTER,
|
2017-09-01 18:29:34 +02:00
|
|
|
($strFile eq MANIFEST_FILE_PGCONTROL || $self->isMasterFile($strFile)));
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
# Link destination required for link type only
|
|
|
|
if ($cType eq 'l')
|
|
|
|
{
|
2016-11-30 21:36:39 +02:00
|
|
|
my $strLinkDestination = $hManifest->{$strName}{link_destination};
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set($strSection, $strFile, MANIFEST_SUBKEY_DESTINATION, $strLinkDestination);
|
|
|
|
|
|
|
|
# If this is a tablespace then set the filter to use for the next level
|
|
|
|
my $strFilter;
|
2015-02-01 01:15:08 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
if ($bTablespace)
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
2017-06-27 22:47:40 +02:00
|
|
|
# Only versions >= 9.0 have the special top-level tablespace path. Below 9.0 the database files are stored
|
|
|
|
# directly in the path referenced by the symlink.
|
2017-09-01 18:29:34 +02:00
|
|
|
if ($self->dbVersion() >= PG_VERSION_90)
|
2017-06-27 22:47:40 +02:00
|
|
|
{
|
|
|
|
$strFilter = $self->tablespacePathGet();
|
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGTBLSPC, undef,
|
|
|
|
$self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA));
|
2015-02-01 01:15:08 +02:00
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
# PGDATA prefix was only needed for the link so strip it off before recursing
|
|
|
|
$strFile = substr($strFile, length(MANIFEST_TARGET_PGDATA) + 1);
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
2018-11-01 17:31:25 +02:00
|
|
|
$bDelta = $self->build(
|
|
|
|
$oStorageDbMaster, $strLinkDestination, undef, $bOnline, $bDelta, $hTablespaceMap, $hDatabaseMap, $rhExclude, undef,
|
|
|
|
undef, $strFile, $bTablespace, dirname("${strPath}/${strName}"), $strFilter, $iLevel + 1);
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# If this is the base level then do post-processing
|
2016-04-15 04:50:02 +02:00
|
|
|
if ($strLevel eq MANIFEST_TARGET_PGDATA)
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
|
|
|
my $bTimeInFuture = false;
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
# Wait for the remainder of the second when doing online backups. This is done because most filesystems only have a one
|
|
|
|
# second resolution and Postgres will still be modifying files during the second that the manifest is built and this could
|
|
|
|
# lead to an invalid diff/incr backup later when using timestamps to determine which files have changed. Offline backups do
|
|
|
|
# not wait because it makes testing much faster and Postgres should not be running (if it is the backup will not be
|
|
|
|
# consistent anyway and the one-second resolution problem is the least of our worries).
|
2019-12-14 00:55:41 +02:00
|
|
|
my $lTimeBegin = waitRemainder($bOnline);
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
if (defined($oLastManifest))
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_PRIOR, undef,
|
|
|
|
$oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
|
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
|
2016-05-11 15:21:39 +02:00
|
|
|
# Store database map information when provided during an online backup.
|
2016-11-30 21:36:39 +02:00
|
|
|
foreach my $strDbName (sort(keys(%{$hDatabaseMap})))
|
2016-05-11 15:21:39 +02:00
|
|
|
{
|
|
|
|
$self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_ID,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_ID});
|
2016-05-11 15:21:39 +02:00
|
|
|
$self->numericSet(MANIFEST_SECTION_DB, $strDbName, MANIFEST_KEY_DB_LAST_SYSTEM_ID,
|
2016-11-30 21:36:39 +02:00
|
|
|
$hDatabaseMap->{$strDbName}{&MANIFEST_KEY_DB_LAST_SYSTEM_ID});
|
2016-05-11 15:21:39 +02:00
|
|
|
}
|
|
|
|
|
2018-11-01 17:31:25 +02:00
|
|
|
# Determine if delta checksum should be enabled
|
|
|
|
if (!$bDelta)
|
|
|
|
{
|
|
|
|
my @stryFileList = $self->keys(MANIFEST_SECTION_TARGET_FILE);
|
|
|
|
|
|
|
|
if (@stryFileList)
|
|
|
|
{
|
|
|
|
$bDelta = $self->checkDeltaFile(\@stryFileList, $oLastManifest, $lTimeBegin);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
# Loop though all files
|
|
|
|
foreach my $strName ($self->keys(MANIFEST_SECTION_TARGET_FILE))
|
|
|
|
{
|
|
|
|
# If modification time is in the future (in this backup OR the last backup) set warning flag and do not
|
|
|
|
# allow a reference
|
|
|
|
if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin ||
|
|
|
|
(defined($oLastManifest) &&
|
|
|
|
$oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y')))
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$bTimeInFuture = true;
|
|
|
|
|
|
|
|
# Only mark as future if still in the future in the current backup
|
|
|
|
if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) > $lTimeBegin)
|
2015-02-01 01:15:08 +02:00
|
|
|
{
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_FUTURE, 'y');
|
|
|
|
}
|
|
|
|
}
|
2018-09-19 17:12:45 +02:00
|
|
|
# Else check if the size and timestamp match OR if the size matches and the delta option is set, then keep the file.
|
|
|
|
# In the latter case, if there had been a timestamp change then rather than removing and recopying the file, the file
|
|
|
|
# will be tested in backupFile to see if the db/repo checksum still matches: if so, it is not necessary to recopy,
|
|
|
|
# else it will need to be copied to the new backup. For zero sized files, the reference will be set and copying
|
|
|
|
# will be skipped later.
|
2016-04-15 04:50:02 +02:00
|
|
|
elsif (defined($oLastManifest) && $oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName) &&
|
|
|
|
$self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) ==
|
2018-09-19 17:12:45 +02:00
|
|
|
$oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) &&
|
|
|
|
($bDelta || ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_SIZE) == 0 ||
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP) ==
|
2018-09-19 17:12:45 +02:00
|
|
|
$oLastManifest->numericGet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_TIMESTAMP))))
|
2016-04-15 04:50:02 +02:00
|
|
|
{
|
|
|
|
# Copy reference from previous backup if possible
|
|
|
|
if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE,
|
2018-11-01 17:31:25 +02:00
|
|
|
$oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE));
|
2016-04-15 04:50:02 +02:00
|
|
|
}
|
|
|
|
# Otherwise the reference is to the previous backup
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REFERENCE,
|
2018-11-01 17:31:25 +02:00
|
|
|
$oLastManifest->get(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_LABEL));
|
2016-04-15 04:50:02 +02:00
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
|
2018-09-19 17:12:45 +02:00
|
|
|
# Copy the checksum from previous manifest (if it exists - zero sized files don't have checksums)
|
2016-04-15 04:50:02 +02:00
|
|
|
if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM,
|
|
|
|
$oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM));
|
|
|
|
}
|
|
|
|
|
2018-09-19 17:12:45 +02:00
|
|
|
# Copy repo size from the previous manifest (if it exists)
|
2016-04-15 04:50:02 +02:00
|
|
|
if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE,
|
2018-11-01 17:31:25 +02:00
|
|
|
$oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_REPO_SIZE));
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
2016-09-04 15:19:44 +02:00
|
|
|
|
|
|
|
# Copy master flag from the previous manifest (if it exists)
|
|
|
|
if ($oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_MASTER))
|
|
|
|
{
|
|
|
|
$self->set(
|
|
|
|
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_MASTER,
|
|
|
|
$oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_MASTER));
|
|
|
|
}
|
2016-12-13 01:54:07 +02:00
|
|
|
|
|
|
|
# Copy checksum page from the previous manifest (if it exists)
|
|
|
|
my $bChecksumPage = $oLastManifest->get(
|
|
|
|
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, false);
|
|
|
|
|
|
|
|
if (defined($bChecksumPage))
|
|
|
|
{
|
|
|
|
$self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE, $bChecksumPage);
|
|
|
|
|
|
|
|
if (!$bChecksumPage &&
|
|
|
|
$oLastManifest->test(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR))
|
|
|
|
{
|
|
|
|
$self->set(
|
|
|
|
MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR,
|
|
|
|
$oLastManifest->get(MANIFEST_SECTION_TARGET_FILE, $strName, MANIFEST_SUBKEY_CHECKSUM_PAGE_ERROR));
|
|
|
|
}
|
|
|
|
}
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Warn if any files in the current backup are in the future
|
|
|
|
if ($bTimeInFuture)
|
|
|
|
{
|
|
|
|
&log(WARN, "some files have timestamps in the future - they will be copied to prevent possible race conditions");
|
|
|
|
}
|
|
|
|
|
|
|
|
# Record the time when copying will start
|
2016-04-15 04:50:02 +02:00
|
|
|
$self->set(MANIFEST_SECTION_BACKUP, MANIFEST_KEY_TIMESTAMP_COPY_START, undef, $lTimeBegin + ($bOnline ? 1 : 0));
|
2015-08-29 20:20:46 +02:00
|
|
|
|
2016-06-27 03:01:20 +02:00
|
|
|
# Build default sections
|
|
|
|
$self->buildDefault();
|
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
2015-08-29 20:20:46 +02:00
|
|
|
# Return from function and log return values if any
|
2018-11-01 17:31:25 +02:00
|
|
|
return logDebugReturn
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
{name => 'bDelta', value => $bDelta, trace => true},
|
|
|
|
);
|
2016-04-15 04:50:02 +02:00
|
|
|
}
|
|
|
|
|
2016-05-14 16:39:56 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# fileAdd
|
|
|
|
#
|
|
|
|
# Add files to the manifest that were generated after the initial manifest build, e.g. backup_label, tablespace_map, and copied WAL
|
|
|
|
# files. Since the files were not in the original cluster the user, group, and mode must be defaulted.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub fileAdd
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my
|
|
|
|
(
|
|
|
|
$strOperation,
|
|
|
|
$strManifestFile,
|
|
|
|
$lModificationTime,
|
|
|
|
$lSize,
|
2016-09-04 15:19:44 +02:00
|
|
|
$strChecksum,
|
|
|
|
$bMaster,
|
2016-05-14 16:39:56 +02:00
|
|
|
) =
|
|
|
|
logDebugParam
|
|
|
|
(
|
|
|
|
__PACKAGE__ . '->fileAdd', \@_,
|
|
|
|
{name => 'strManifestFile'},
|
|
|
|
{name => 'lModificationTime'},
|
|
|
|
{name => 'lSize'},
|
2016-09-04 15:19:44 +02:00
|
|
|
{name => 'lChecksum'},
|
|
|
|
{name => 'bMaster'},
|
2016-05-14 16:39:56 +02:00
|
|
|
);
|
|
|
|
|
|
|
|
# Set manifest values
|
2016-06-27 03:01:20 +02:00
|
|
|
if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER) ||
|
|
|
|
!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_USER, undef,
|
|
|
|
$self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER)))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_USER,
|
|
|
|
$self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_USER));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP) ||
|
|
|
|
!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_GROUP, undef,
|
|
|
|
$self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP)))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_GROUP,
|
|
|
|
$self->get(MANIFEST_SECTION_TARGET_PATH, MANIFEST_TARGET_PGDATA, MANIFEST_SUBKEY_GROUP));
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE) ||
|
|
|
|
!$self->test(MANIFEST_SECTION_TARGET_FILE . ':default', MANIFEST_SUBKEY_MODE, undef, '0600'))
|
|
|
|
{
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MODE, '0600');
|
|
|
|
}
|
|
|
|
|
2016-05-14 16:39:56 +02:00
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_TIMESTAMP, $lModificationTime);
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_SIZE, $lSize);
|
|
|
|
$self->set(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_CHECKSUM, $strChecksum);
|
2016-09-04 15:19:44 +02:00
|
|
|
$self->boolSet(MANIFEST_SECTION_TARGET_FILE, $strManifestFile, MANIFEST_SUBKEY_MASTER, $bMaster);
|
2016-05-14 16:39:56 +02:00
|
|
|
}
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# buildDefault
|
|
|
|
#
|
|
|
|
# Builds the default section.
|
|
|
|
####################################################################################################################################
|
|
|
|
sub buildDefault
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
2016-08-11 23:32:28 +02:00
|
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . '->buildDefault');
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
# Set defaults for subkeys that tend to repeat
|
|
|
|
foreach my $strSection (&MANIFEST_SECTION_TARGET_FILE, &MANIFEST_SECTION_TARGET_PATH, &MANIFEST_SECTION_TARGET_LINK)
|
|
|
|
{
|
2016-09-04 15:19:44 +02:00
|
|
|
foreach my $strSubKey (&MANIFEST_SUBKEY_USER, &MANIFEST_SUBKEY_GROUP, &MANIFEST_SUBKEY_MODE, &MANIFEST_SUBKEY_MASTER)
|
2016-04-15 04:50:02 +02:00
|
|
|
{
|
2016-06-27 03:01:20 +02:00
|
|
|
# Links don't have a mode so skip
|
|
|
|
next if ($strSection eq MANIFEST_SECTION_TARGET_LINK && $strSubKey eq &MANIFEST_SUBKEY_MODE);
|
|
|
|
|
2016-09-04 15:19:44 +02:00
|
|
|
# Only files have the master subkey
|
|
|
|
next if ($strSection ne MANIFEST_SECTION_TARGET_FILE && $strSubKey eq &MANIFEST_SUBKEY_MASTER);
|
|
|
|
|
2016-04-15 04:50:02 +02:00
|
|
|
my %oDefault;
|
|
|
|
my $iSectionTotal = 0;
|
|
|
|
|
|
|
|
foreach my $strFile ($self->keys($strSection))
|
|
|
|
{
|
2016-09-04 15:19:44 +02:00
|
|
|
# Don't count false values when subkey in (MANIFEST_SUBKEY_USER, MANIFEST_SUBKEY_GROUP)
|
|
|
|
next if (($strSubKey eq MANIFEST_SUBKEY_USER || $strSubKey eq MANIFEST_SUBKEY_GROUP) &&
|
|
|
|
$self->boolTest($strSection, $strFile, $strSubKey, false));
|
2016-06-27 03:01:20 +02:00
|
|
|
|
2016-09-04 15:19:44 +02:00
|
|
|
my $strValue = $self->get($strSection, $strFile, $strSubKey);
|
|
|
|
|
|
|
|
if (defined($oDefault{$strValue}))
|
|
|
|
{
|
|
|
|
$oDefault{$strValue}++;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$oDefault{$strValue} = 1;
|
2016-04-15 04:50:02 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
$iSectionTotal++;
|
|
|
|
}
|
|
|
|
|
|
|
|
my $strMaxValue;
|
|
|
|
my $iMaxValueTotal = 0;
|
|
|
|
|
2017-11-28 18:44:24 +02:00
|
|
|
foreach my $strValue (sort(keys(%oDefault)))
|
2016-04-15 04:50:02 +02:00
|
|
|
{
|
|
|
|
if ($oDefault{$strValue} > $iMaxValueTotal)
|
|
|
|
{
|
|
|
|
$iMaxValueTotal = $oDefault{$strValue};
|
|
|
|
$strMaxValue = $strValue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (defined($strMaxValue) > 0 && $iMaxValueTotal > $iSectionTotal * MANIFEST_DEFAULT_MATCH_FACTOR)
|
|
|
|
{
|
2016-09-04 15:19:44 +02:00
|
|
|
if ($strSubKey eq MANIFEST_SUBKEY_MASTER)
|
|
|
|
{
|
|
|
|
$self->boolSet("${strSection}:default", $strSubKey, undef, $strMaxValue);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
$self->set("${strSection}:default", $strSubKey, undef, $strMaxValue);
|
|
|
|
}
|
2016-04-15 04:50:02 +02:00
|
|
|
|
|
|
|
foreach my $strFile ($self->keys($strSection))
|
|
|
|
{
|
|
|
|
if ($self->test($strSection, $strFile, $strSubKey, $strMaxValue))
|
|
|
|
{
|
|
|
|
$self->remove($strSection, $strFile, $strSubKey);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn($strOperation);
|
2015-02-01 01:15:08 +02:00
|
|
|
}
|
|
|
|
|
2016-09-06 15:35:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# validate
|
|
|
|
#
|
2019-08-26 18:05:36 +02:00
|
|
|
# Checks for any missing values or inconsistencies in the manifest.
|
2016-09-06 15:35:02 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
sub validate
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
# Assign function parameters, defaults, and log debug info
|
|
|
|
my ($strOperation) = logDebugParam(__PACKAGE__ . 'validate');
|
|
|
|
|
|
|
|
# Make sure that all files have size and checksum (when size > 0). Since these values are removed before the backup file copy
|
|
|
|
# starts this ensures that all files had results stored in the manifest during the file copy.
|
|
|
|
foreach my $strFile ($self->keys(MANIFEST_SECTION_TARGET_FILE))
|
|
|
|
{
|
|
|
|
# Ensure size is set
|
|
|
|
if (!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE))
|
|
|
|
{
|
|
|
|
confess &log(ASSERT, "manifest subvalue 'size' not set for file '${strFile}'");
|
|
|
|
}
|
|
|
|
|
|
|
|
# If size > 0 then checksum must also be set
|
|
|
|
if ($self->numericGet(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_SIZE) > 0 &&
|
|
|
|
!$self->test(MANIFEST_SECTION_TARGET_FILE, $strFile, MANIFEST_SUBKEY_CHECKSUM))
|
|
|
|
{
|
|
|
|
confess &log(ASSERT, "manifest subvalue 'checksum' not set for file '${strFile}'");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
# Return from function and log return values if any
|
|
|
|
return logDebugReturn($strOperation);
|
|
|
|
}
|
|
|
|
|
2017-09-01 18:29:34 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# dbVersion - version of PostgreSQL that the manifest is being built for
|
|
|
|
####################################################################################################################################
|
|
|
|
sub dbVersion
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
return $self->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION);
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# xactPath - return the transaction directory based on the PostgreSQL version
|
|
|
|
####################################################################################################################################
|
|
|
|
sub xactPath
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
return $self->dbVersion() >= PG_VERSION_10 ? 'pg_xact' : 'pg_clog';
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# walPath - return the wal directory based on the PostgreSQL version
|
|
|
|
####################################################################################################################################
|
|
|
|
sub walPath
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
|
|
|
|
return $self->dbVersion() >= PG_VERSION_10 ? 'pg_wal' : 'pg_xlog';
|
|
|
|
}
|
|
|
|
|
|
|
|
####################################################################################################################################
|
|
|
|
# isMasterFile
|
|
|
|
#
|
2017-11-28 18:44:24 +02:00
|
|
|
# Is this file required to be copied from the master?
|
2017-09-01 18:29:34 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
sub isMasterFile
|
|
|
|
{
|
|
|
|
my $self = shift;
|
|
|
|
my $strFile = shift;
|
|
|
|
|
|
|
|
return
|
|
|
|
$strFile !~ ('^(' . MANIFEST_TARGET_PGDATA . '\/' . '(' . DB_PATH_BASE . '|' . DB_PATH_GLOBAL . '|' .
|
|
|
|
$self->xactPath() . '|' . DB_PATH_PGMULTIXACT . ')|' . DB_PATH_PGTBLSPC . ')\/');
|
|
|
|
}
|
|
|
|
|
2016-12-13 01:54:07 +02:00
|
|
|
####################################################################################################################################
|
|
|
|
# isChecksumPage
|
|
|
|
#
|
|
|
|
# Can this file have page checksums in PG >= 9.3?
|
|
|
|
####################################################################################################################################
|
|
|
|
sub isChecksumPage
|
|
|
|
{
|
|
|
|
my $strFile = shift;
|
|
|
|
|
2019-11-25 14:37:09 +02:00
|
|
|
if (($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_BASE . '\/[0-9]+\/|^' . MANIFEST_TARGET_PGTBLSPC .
|
|
|
|
'\/[0-9]+\/[^\/]+\/[0-9]+\/') &&
|
2016-12-13 01:54:07 +02:00
|
|
|
$strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . ')$')) ||
|
2019-11-25 14:37:09 +02:00
|
|
|
($strFile =~ ('^' . MANIFEST_TARGET_PGDATA . '\/' . DB_PATH_GLOBAL . '\/') &&
|
2016-12-13 01:54:07 +02:00
|
|
|
$strFile !~ ('(' . DB_FILE_PGFILENODEMAP . '|' . DB_FILE_PGINTERNALINIT . '|' . DB_FILE_PGVERSION . '|' .
|
2017-03-02 20:50:29 +02:00
|
|
|
DB_FILE_PGCONTROL . ')$')))
|
2016-12-13 01:54:07 +02:00
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
push @EXPORT, qw(isChecksumPage);
|
|
|
|
|
2015-01-07 17:59:43 +02:00
|
|
|
1;
|