From 4e7bd4468a785a00c37b5d29262d86f76f725a78 Mon Sep 17 00:00:00 2001 From: David Steele Date: Thu, 6 Aug 2015 16:36:55 -0400 Subject: [PATCH] Worked on issue #122: 9.5 Integration. Most tests are working now. What's not working: 1) --target-resume option fails because pause_on_recovery setting was removed. Need to implement to the new 9.5 option and make that work with older versions in a consistent way. 2) No tests for the new .partial WAL segments that can be generated on timeline switch. --- README.md | 4 +++- doc/doc.xml | 5 ++++- lib/BackRest/Restore.pm | 13 +++++++++++-- test/lib/BackRestTest/BackupCommonTest.pm | 14 ++++++++++---- test/lib/BackRestTest/BackupTest.pm | 2 +- test/lib/BackRestTest/CommonTest.pm | 17 ++++++++--------- test/vm/Vagrantfile | 8 +++++--- 7 files changed, 42 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 28484235a..8ef965002 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ pgBackRest uses the gitflow model of development. This means that the master br ## Install -pgBackRest is written entirely in Perl and uses some non-standard modules that must be installed from CPAN. +pgBackRest is written entirely in Perl. Some additional modules will need to be installed depending on the OS. ### Ubuntu 12.04/14.04 Setup @@ -796,6 +796,8 @@ Get information about backups in the `db` stanza. * Added vagrant test configurations for Ubuntu 14.04 and CentOS 7. +* Experimental support for PostgreSQL 9.5 alpha1. This may break when the control version or WAL magic changes in future versions but will be updated in each pgBackRest release to keep pace. All regression tests pass except for `--target-resume` tests (this functionality has changed in 9.5) and there is no testing yet for `.partial` WAL segments. + ### v0.78: Remove CPAN dependencies, stability improvements * Removed dependency on CPAN packages for multi-threaded operation. While it might not be a bad idea to update the `threads` and `Thread::Queue` packages, it is no longer necessary. diff --git a/doc/doc.xml b/doc/doc.xml index bfd1eaee7..cc95fbe0b 100644 --- a/doc/doc.xml +++ b/doc/doc.xml @@ -28,7 +28,7 @@ - is written entirely in Perl and uses some non-standard modules that must be installed from CPAN. + is written entirely in Perl. Some additional modules will need to be installed depending on the OS. @@ -756,6 +756,9 @@ Run a full backup on the db stanza. --type can Added vagrant test configurations for Ubuntu 14.04 and CentOS 7. + + Experimental support for 9.5 alpha1. This may break when the control version or WAL magic changes in future versions but will be updated in each release to keep pace. All regression tests pass except for --target-resume tests (this functionality has changed in 9.5) and there is no testing yet for .partial WAL segments. + diff --git a/lib/BackRest/Restore.pm b/lib/BackRest/Restore.pm index 75d4129bb..52bd17fb9 100644 --- a/lib/BackRest/Restore.pm +++ b/lib/BackRest/Restore.pm @@ -24,9 +24,10 @@ use BackRest::ThreadGroup; use BackRest::Utility; #################################################################################################################################### -# Recovery.conf file +# File constants #################################################################################################################################### -use constant FILE_RECOVERY_CONF => 'recovery.conf'; +use constant FILE_RECOVERY_CONF => 'recovery.conf'; # Conf file with settings for recovery after restore +use constant FILE_TABLESPACE_MAP => 'tablespace_map'; # Tablespace map introduced in 9.5 #################################################################################################################################### # CONSTRUCTOR @@ -743,6 +744,14 @@ sub restore } } + # Remove the tablespace_map file in versions >= 9.5 so Postgres does not rewrite the links in the pg_tblspc directory. + if ($oManifest->get(MANIFEST_SECTION_BACKUP_DB, MANIFEST_KEY_DB_VERSION) >= 9.5) + { + $self->{oFile}->remove(PATH_DB_ABSOLUTE, + $oManifest->get(MANIFEST_SECTION_BACKUP_PATH, MANIFEST_KEY_BASE, MANIFEST_SUBKEY_PATH) . + '/' . FILE_TABLESPACE_MAP); + } + # Create recovery.conf file $self->recovery(); diff --git a/test/lib/BackRestTest/BackupCommonTest.pm b/test/lib/BackRestTest/BackupCommonTest.pm index a620aeccd..4e4f2cdfd 100644 --- a/test/lib/BackRestTest/BackupCommonTest.pm +++ b/test/lib/BackRestTest/BackupCommonTest.pm @@ -298,13 +298,12 @@ sub BackRestTestBackup_ClusterStart confess 'postmaster.pid exists'; } - # Creat the archive command + # Create the archive command my $strArchive = BackRestTestCommon_CommandMainAbsGet() . ' --stanza=' . BackRestTestCommon_StanzaGet() . ' --config=' . BackRestTestCommon_DbPathGet() . '/pg_backrest.conf archive-push %p'; # Start the cluster - my $strCommand = BackRestTestCommon_PgSqlBinPathGet() . "/pg_ctl start -o \"-c port=${iPort}" . - ' -c checkpoint_segments=1'; + my $strCommand = BackRestTestCommon_PgSqlBinPathGet() . "/pg_ctl start -o \"-c port=${iPort}"; if ($bArchive) { @@ -330,7 +329,8 @@ sub BackRestTestBackup_ClusterStart $strCommand .= " -c archive_mode=on -c wal_level=archive -c archive_command=true"; } - $strCommand .= " -c unix_socket_director" . (BackRestTestCommon_DbVersion() < '9.3' ? "y='" : "ies='") . + $strCommand .= " -c log_error_verbosity=verbose" . + " -c unix_socket_director" . (BackRestTestCommon_DbVersion() < '9.3' ? "y='" : "ies='") . BackRestTestCommon_DbPathGet() . "'\" " . "-D ${strPath} -l ${strPath}/postgresql.log -s"; @@ -1349,6 +1349,12 @@ sub BackRestTestBackup_RestoreCompare if (!$bSynthetic) { + # Tablespace_map file is not restored in versions >= 9.5 because it interferes with internal remapping features. + if (${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_DB}{&MANIFEST_KEY_DB_VERSION} >= 9.5) + { + delete(${$oExpectedManifestRef}{'base:file'}{'tablespace_map'}); + } + foreach my $strTablespaceName (keys(%{${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_PATH}})) { if (defined(${$oExpectedManifestRef}{&MANIFEST_SECTION_BACKUP_PATH}{$strTablespaceName}{&MANIFEST_SUBKEY_LINK})) diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index b1c5378e5..36b8d856e 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -1334,7 +1334,7 @@ sub BackRestTestBackup_Test $strType = RECOVERY_TYPE_XID; $strTarget = $strXidTarget; $bTargetExclusive = undef; - $bTargetResume = BackRestTestCommon_DbVersion() >= 9.1 ? true : undef; + $bTargetResume = BackRestTestCommon_DbVersion() >= 9.1 && BackRestTestCommon_DbVersion() < 9.5 ? true : undef; $strTargetTimeline = undef; $oRecoveryHashRef = undef; $strComment = undef; diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 5507f5e74..bcb0aa9ea 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -791,14 +791,19 @@ sub BackRestTestCommon_Setup BackRestTestCommon_Execute($strPgSqlBin . '/postgres --version'); # Get the Postgres version + my $strVersionRegExp = '(devel|((alpha|beta)[0-9]+))$'; my @stryVersionToken = split(/ /, $strOutLog); @stryVersionToken = split(/\./, $stryVersionToken[2]); $strCommonDbVersion = $stryVersionToken[0] . '.' . trim($stryVersionToken[1]); - if ($strCommonDbVersion =~ /devel$/) + # Warn if this is a devel/alpha/beta version + if ($strCommonDbVersion =~ /$strVersionRegExp/) { - $strCommonDbVersion =~ s/devel$//; - &log(INFO, "Testing against ${strCommonDbVersion} development version"); + my $strDevVersion = $strCommonDbVersion; + $strCommonDbVersion =~ s/$strVersionRegExp//; + $strDevVersion = substr($strDevVersion, length($strCommonDbVersion)); + + &log(WARN, "Testing against ${strCommonDbVersion} ${strDevVersion} version"); } # Don't run unit tests for unsupported versions @@ -809,12 +814,6 @@ sub BackRestTestCommon_Setup confess "currently only version ${$strVersionSupport}[0] and up are supported"; } - if ($strCommonDbVersion eq '9.5') - { - &log(WARN, "unit tests do not currently work with version 9.5"); - return false; - } - return true; } diff --git a/test/vm/Vagrantfile b/test/vm/Vagrantfile index 1aad60ccd..472be08e3 100644 --- a/test/vm/Vagrantfile +++ b/test/vm/Vagrantfile @@ -53,9 +53,11 @@ Vagrant.configure(2) do |config| # Provision the VM u14.vm.provision "shell", inline: <<-SHELL # Install db - echo 'deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main' > /etc/apt/sources.list.d/pgdg.list + echo 'deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main 9.5' >> /etc/apt/sources.list.d/pgdg.list wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update + apt-get install -y postgresql-9.5 + pg_dropcluster --stop 9.5 main apt-get install -y postgresql-9.4 pg_dropcluster --stop 9.4 main apt-get install -y postgresql-9.3 @@ -66,8 +68,6 @@ Vagrant.configure(2) do |config| pg_dropcluster --stop 9.1 main apt-get install -y postgresql-9.0 pg_dropcluster --stop 9.0 main - apt-get install -y postgresql-8.4 - pg_dropcluster --stop 8.4 main # Setup SSH adduser --ingroup=vagrant --disabled-password --gecos "" backrest @@ -124,8 +124,10 @@ Vagrant.configure(2) do |config| # Install db sudo rpm -ivh http://yum.postgresql.org/9.3/redhat/rhel-7-x86_64/pgdg-centos93-9.3-1.noarch.rpm sudo rpm -ivh http://yum.postgresql.org/9.4/redhat/rhel-7-x86_64/pgdg-centos94-9.4-1.noarch.rpm + sudo rpm -ivh http://yum.postgresql.org/9.5/redhat/rhel-7-x86_64/pgdg-centos95-9.5-1.noarch.rpm yum -y install postgresql93-server yum -y install postgresql94-server + yum -y install postgresql95-server # Install Perl and required modules yum -y install perl perl-IO-String perl-Thread-Queue perl-JSON-PP perl-Digest-SHA perl-DBD-Pg