db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-10-30 23:37:45 +02:00
Code Issues Releases Activity

Container build improvements:

Browse Source 
* More optimized container suite that greatly improves build time.
* Added static Debian packages for Devel::Cover to reduce build time.
* Add deprecated state for containers. Deprecated containers may only be used to build packages.
* Remove Debian 8 from CI because it does not provide additional coverage over Ubuntu 14.04 and Ubuntu 16.04.
This commit is contained in:
David Steele
2017-06-24 10:59:00 -04:00
parent 53184a7b30
commit 7a9b1f6993
20 changed files with 429 additions and 728 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
.travis.yml
View File

@@ -16,7 +16,6 @@ services:
env:
- PGB_TEST_VM="co6" PGB_TEST_PARAM=""
- PGB_TEST_VM="u16" PGB_TEST_PARAM=" --no-lint"
- PGB_TEST_VM="d8" PGB_TEST_PARAM=" --no-lint"
- PGB_TEST_VM="co7" PGB_TEST_PARAM=" --no-lint"
- PGB_TEST_VM="u14" PGB_TEST_PARAM=" --no-lint"
@@ -31,10 +30,8 @@ before_install:
aws help --version
aws configure list
- |
# Build Devel::Cover
git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl
cd ~/libdevel-cover-perl && git checkout debian/1.23-2 && debuild -i -us -uc -b
sudo dpkg -i ~/libdevel-cover-perl_1.23-2_amd64.deb
# Install Devel::Cover
sudo dpkg -i ${TRAVIS_BUILD_DIR?}/test/package/u14-libdevel-cover-perl_1.23-2_amd64.deb
/usr/bin/cover -v
install:

18
doc/xml/release.xml
View File

@@ -263,7 +263,23 @@
</release-item>
<release-item>
<p>Remove <proper>Ubuntu 12.04</proper> from CI now that it is EOL (it can still be run manually).</p>
<p>More optimized container suite that greatly improves build time.</p>
</release-item>
<release-item>
<p>Added static Debian packages for <code>Devel::Cover</code> to reduce build time.</p>
</release-item>
<release-item>
<p>Add <id>deprecated</id> state for containers. Deprecated containers may only be used to build packages.</p>
</release-item>
<release-item>
<p>Deprecate <proper>Ubuntu 12.04</proper> now that it is EOL.</p>
</release-item>
<release-item>
<p>Remove <proper>Debian 8</proper> from CI because it does not provide additional coverage over <proper>Ubuntu 14.04</proper> and <proper>Ubuntu 16.04</proper>.</p>
</release-item>
<release-item>

4
doc/xml/user-guide.xml
View File

@@ -71,7 +71,7 @@
<variable key="host-db-master">db-master</variable>
<variable key="host-db-master-user">{[host-user]}</variable>
<variable key="host-db-master-image">{[image-repo]}:{[host-os]}-db-{[pg-version]}-doc</variable>
<variable key="host-db-master-image">{[image-repo]}:{[host-os]}-doc-db</variable>
<variable key="host-db-master-mount">{[host-mount]}</variable>
<variable key="host-db-standby">db-standby</variable>
@@ -81,7 +81,7 @@
<variable key="host-backup">backup</variable>
<variable key="host-backup-user">{[host-user]}</variable>
<variable key="host-backup-image">{[image-repo]}:{[host-os]}-backup-doc-pre</variable>
<variable key="host-backup-image">{[image-repo]}:{[host-os]}-doc-backup</variable>
<variable key="host-backup-mount">{[host-mount]}</variable>
<!-- Commands for various operations -->

1
test/.gitignore vendored
View File

@@ -1,6 +1,5 @@
test
.vagrant
package
nytprof*
scratch.txt
coverage*

8
test/Vagrantfile vendored
View File

@@ -61,12 +61,8 @@ Vagrant.configure(2) do |config|
>> /home/ubuntu/.profile
#---------------------------------------------------------------------------------------------------------------------------
echo 'Build Devel::Cover' && date
apt-get install -y libpod-coverage-perl libtest-differences-perl libhtml-parser-perl libtemplate-perl
git clone --branch debian/1.23-2 \
https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git /root/libdevel-cover-perl
cd /root/libdevel-cover-perl && debuild --no-lintian -i -us -uc -b > /dev/null
dpkg -i /root/libdevel-cover-perl_1.23-2_amd64.deb
echo 'Install Devel::Cover' && date
dpkg -i /backrest/test/package/u16-libdevel-cover-perl_1.23-2_amd64.deb
#---------------------------------------------------------------------------------------------------------------------------
echo 'Install Docker' && date

6
test/lib/pgBackRestTest/Common/CiTest.pm
View File

@@ -115,10 +115,8 @@ sub process
" aws help --version\n" .
" aws configure list\n" .
" - |\n" .
" # Build Devel::Cover\n" .
" git clone https://anonscm.debian.org/git/pkg-perl/packages/libdevel-cover-perl.git ~/libdevel-cover-perl\n" .
' cd ~/libdevel-cover-perl && git checkout debian/' . LIB_COVER_VERSION . " && debuild -i -us -uc -b\n" .
' sudo dpkg -i ~/' . LIB_COVER_PACKAGE . "\n" .
" # Install Devel::Cover\n" .
" sudo dpkg -i \${TRAVIS_BUILD_DIR?}/test/package/u14-" . LIB_COVER_PACKAGE . "\n" .
' ' . LIB_COVER_EXE . " -v\n" .
"\n" .
"install:\n" .

993
test/lib/pgBackRestTest/Common/ContainerTest.pm
View File

File diff suppressed because it is too large Load Diff

2
test/lib/pgBackRestTest/Common/DefineTest.pm
View File

@@ -213,7 +213,7 @@ my $oTestDef =
{
&TESTDEF_NAME => 's3',
&TESTDEF_TOTAL => 7,
&TESTDEF_VM => [VM_CO7, VM_U16],
&TESTDEF_VM => [VM_CO7, VM_U14, VM_U16, VM_D8],
&TESTDEF_COVERAGE =>
{

2
test/lib/pgBackRestTest/Common/JobTest.pm
View File

@@ -149,7 +149,7 @@ sub run
" -v ${strHostTestPath}:${strVmTestPath}" .
" -v $self->{strBackRestBase}:$self->{strBackRestBase} " .
containerRepo() . ':' . $self->{oTest}->{&TEST_VM} .
"-loop-test-pre",
"-test",
{bSuppressStdErr => true});
}
}

4
test/lib/pgBackRestTest/Common/ListTest.pm
View File

@@ -83,10 +83,10 @@ sub testListGet
my $iDbVersionMax = -1;
# By default test every db version that is supported for each OS
my $strDbVersionKey = 'db';
my $strDbVersionKey = &VM_DB;
# Run a reduced set of tests where each PG version is only tested on a single OS
if ($strDbVersion eq 'minimal')
if ($strDbVersion eq 'minimal' && defined($oyVm->{$strTestOS}{&VM_DB_MINIMAL}))
{
$strDbVersionKey = &VM_DB_MINIMAL;
}

92
test/lib/pgBackRestTest/Common/VmTest.pm
View File

@@ -21,12 +21,12 @@ use pgBackRest::DbVersion;
####################################################################################################################################
use constant VM_DB => 'db';
push @EXPORT, qw(VM_DB);
use constant VM_DB_DOC => 'db-doc';
push @EXPORT, qw(VM_DB_DOC);
use constant VM_DB_MINIMAL => 'db-minimal';
push @EXPORT, qw(VM_DB_MINIMAL);
use constant VM_CONTROL_MASTER => 'control-master';
push @EXPORT, qw(VM_CONTROL_MASTER);
use constant VM_DEPRECATED => 'deprecated';
push @EXPORT, qw(VM_DEPRECATED);
use constant VM_IMAGE => 'image';
push @EXPORT, qw(VM_IMAGE);
use constant VM_OS => 'os';
@@ -82,7 +82,7 @@ use constant VM_HOST_DEFAULT => VM_U16;
push @EXPORT, qw(VM_HOST_DEFAULT);
# Lists valid VMs
use constant VM_LIST => (VM_CO6, VM_U16, VM_D8, VM_CO7, VM_U14);
use constant VM_LIST => (VM_CO6, VM_U16, VM_CO7, VM_U14);
push @EXPORT, qw(VM_LIST);
my $oyVm =
@@ -93,7 +93,6 @@ my $oyVm =
&VM_OS_BASE => VM_OS_BASE_RHEL,
&VM_OS => VM_OS_CENTOS,
&VM_IMAGE => 'centos:6',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib64/perl5',
@@ -101,9 +100,6 @@ my $oyVm =
[
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
@@ -114,11 +110,6 @@ my $oyVm =
PG_VERSION_91,
PG_VERSION_95,
],
&VM_DB_DOC =>
[
PG_VERSION_95,
],
},
# CentOS 7
@@ -127,19 +118,10 @@ my $oyVm =
&VM_OS_BASE => VM_OS_BASE_RHEL,
&VM_OS => VM_OS_CENTOS,
&VM_IMAGE => 'centos:7',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/pgsql-{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib64/perl5',
&VM_DB =>
[
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
&VM_DB_MINIMAL =>
[
PG_VERSION_96,
],
@@ -152,56 +134,20 @@ my $oyVm =
&VM_OS => VM_OS_DEBIAN,
&VM_OS_REPO => 'jessie',
&VM_IMAGE => 'debian:8',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.20.2',
&VM_DB =>
[
PG_VERSION_84,
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
&VM_DB_MINIMAL =>
[
PG_VERSION_84,
PG_VERSION_92,
],
},
# Ubuntu 12.04 (no longer included in normal testing, but here if needed)
# Ubuntu 12.04 (DEPRECATED)
&VM_U12 =>
{
&VM_DEPRECATED => true,
&VM_OS_BASE => VM_OS_BASE_DEBIAN,
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'precise',
&VM_IMAGE => 'ubuntu:12.04',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/perl/5.14.2',
&VM_DB =>
[
PG_VERSION_84,
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
&VM_DB_MINIMAL =>
[
PG_VERSION_84,
],
},
# Ubuntu 14.04
@@ -211,28 +157,12 @@ my $oyVm =
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'trusty',
&VM_IMAGE => 'ubuntu:14.04',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/perl/5.18.2',
&VM_DB =>
[
PG_VERSION_90,
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
&VM_DB_MINIMAL =>
[
PG_VERSION_94,
],
&VM_DB_DOC =>
[
PG_VERSION_84,
PG_VERSION_94,
],
},
@@ -244,23 +174,13 @@ my $oyVm =
&VM_OS => VM_OS_UBUNTU,
&VM_OS_REPO => 'xenial',
&VM_IMAGE => 'ubuntu:16.04',
&VM_CONTROL_MASTER => false,
&VMDEF_PGSQL_BIN => '/usr/lib/postgresql/{[version]}/bin',
&VMDEF_PERL_ARCH_PATH => '/usr/local/lib/x86_64-linux-gnu/perl/5.22.1',
&VM_DB =>
[
PG_VERSION_91,
PG_VERSION_92,
PG_VERSION_93,
PG_VERSION_94,
PG_VERSION_95,
PG_VERSION_96,
],
&VM_DB_MINIMAL =>
[
PG_VERSION_93,
],
},
};

2
test/lib/pgBackRestTest/Env/Host/HostBackupTest.pm
View File

@@ -75,7 +75,7 @@ sub new
if (!defined($$oParam{strName}) || $$oParam{strName} eq HOST_BACKUP)
{
$strName = HOST_BACKUP;
$strImage = containerRepo() . ':' . testRunGet()->vm() . '-backup-test-pre';
$strImage = containerRepo() . ':' . testRunGet()->vm() . '-test';
$strUser = testRunGet()->backrestUser();
}
else

2
test/lib/pgBackRestTest/Env/Host/HostDbSyntheticTest.pm
View File

@@ -55,7 +55,7 @@ sub new
my $self = $class->SUPER::new(
{
strImage => containerRepo() . ':' . testRunGet()->vm() . "-db-test-pre",
strImage => containerRepo() . ':' . testRunGet()->vm() . "-test",
strBackupDestination => $$oParam{strBackupDestination},
oLogTest => $$oParam{oLogTest},
bSynthetic => true,

2
test/lib/pgBackRestTest/Env/Host/HostDbTest.pm
View File

@@ -70,7 +70,7 @@ sub new
my $self = $class->SUPER::new(
{
strImage => containerRepo() . ':' . testRunGet()->vm() . "-db-${strDbVersion}-test-pre",
strImage => containerRepo() . ':' . testRunGet()->vm() . "-test",
strBackupDestination => $$oParam{strBackupDestination},
oLogTest => $$oParam{oLogTest},
bStandby => $$oParam{bStandby},

2
test/lib/pgBackRestTest/Env/S3EnvTest.pm
View File

@@ -56,7 +56,7 @@ sub initS3
return new pgBackRest::Storage::S3::Driver(
$strBucket, $strEndPoint, $strRegion, $strAccessKeyId, $strSecretAccessKey,
{strCaFile => $self->vm() eq VM_CO7 ? '/root/scalitys3/ca.crt' : undef,
bVerifySsl => $self->vm() eq VM_U16 ? false : undef, lBufferMax => 1048576});
bVerifySsl => $self->vm() eq VM_CO7 ? undef : false, lBufferMax => 1048576});
}
1;

4
test/lib/pgBackRestTest/Module/Real/RealAllTest.pm
View File

@@ -66,10 +66,10 @@ sub run
my $hyVm = vmGet();
if (($bS3 || $bHostBackup) &&
(@{$hyVm->{$self->vm()}{&VM_DB_MINIMAL}} > 1 && ${$hyVm->{$self->vm()}{&VM_DB_MINIMAL}}[-1] ne $self->pgVersion()))
(@{$hyVm->{$self->vm()}{&VM_DB}} > 1 && ${$hyVm->{$self->vm()}{&VM_DB}}[-1] ne $self->pgVersion()))
{
&log(INFO,
'skipped - this test will be run for this OS using PG ' . ${$hyVm->{$self->vm()}{&VM_DB_MINIMAL}}[-1]);
'skipped - this test will be run for this OS using PG ' . ${$hyVm->{$self->vm()}{&VM_DB}}[-1]);
next;
}

BIN
test/package/d8-libdevel-cover-perl_1.23-2_amd64.deb Normal file
View File

Binary file not shown.

BIN
test/package/u14-libdevel-cover-perl_1.23-2_amd64.deb Normal file
View File

Binary file not shown.

BIN
test/package/u16-libdevel-cover-perl_1.23-2_amd64.deb Normal file
View File

Binary file not shown.

8
test/test.pl
View File

@@ -76,8 +76,8 @@ test.pl [options]
--smart perform libc/package builds only when source timestamps have changed
--no-package do not build packages
--no-ci-config don't overwrite the current continuous integration config
--dev --no-lint --smart --no-package --process-max=1
--expect --no-lint --smart --no-package --process-max=1 --vm=co7 --db=9.6 --log-force
--dev --no-lint --smart --no-package
--expect --no-lint --smart --no-package --vm=co7 --db=9.6 --log-force
Configuration Options:
--psql-bin path to the psql executables (e.g. /usr/lib/postgresql/9.3/bin/)
@@ -276,7 +276,7 @@ eval
################################################################################################################################
if ($bVmBuild)
{
containerBuild($oStorageBackRest, $strVm, $bVmForce, $strDbVersion);
containerBuild($oStorageBackRest, $strVm, $bVmForce);
exit 0;
}
@@ -550,7 +550,7 @@ eval
"bash -c 'cp -r /root/package-src/debian ${strBuildPath}' && sudo chown -R " . TEST_USER .
" ${strBuildPath}");
# If dev build then override then disable static release date used for reproducibility.
# If dev build then disable static release date used for reproducibility
if ($bVersionDev)
{
my $strRules = ${$oStorageBackRest->get("${strBuildPath}/debian/rules")};