From 835396751f220be3f8e46f344d4eadfd7b7305b3 Mon Sep 17 00:00:00 2001 From: David Steele Date: Sun, 10 Jun 2018 14:13:56 -0400 Subject: [PATCH] Document generator improvements. * Build containers from scratch for more accurate testing. * Allow environment load to be skipped. * Allow bash wrapping to be skipped. * Allow forcing a command to run as a user without sudo. --- doc/lib/BackRestDoc/Common/DocExecute.pm | 13 +- doc/xml/dtd/doc.dtd | 3 + doc/xml/release.xml | 6 +- doc/xml/user-guide.xml | 648 +++++++++++++----- .../pgBackRestTest/Common/HostGroupTest.pm | 9 +- test/lib/pgBackRestTest/Common/HostTest.pm | 23 +- test/travis.pl | 8 - 7 files changed, 519 insertions(+), 191 deletions(-) diff --git a/doc/lib/BackRestDoc/Common/DocExecute.pm b/doc/lib/BackRestDoc/Common/DocExecute.pm index 1217adef8..57f73004e 100644 --- a/doc/lib/BackRestDoc/Common/DocExecute.pm +++ b/doc/lib/BackRestDoc/Common/DocExecute.pm @@ -154,9 +154,10 @@ sub executeKey ); # Add user to command + my $bUserForce = $oCommand->paramTest('user-force', 'y') ? true : false; my $strCommand = $self->{oManifest}->variableReplace(trim($oCommand->fieldGet('exe-cmd'))); - my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, 'postgres')); - $strCommand = ($strUser eq DOC_USER ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand; + my $strUser = $self->{oManifest}->variableReplace($oCommand->paramGet('user', false, DOC_USER)); + $strCommand = ($strUser eq DOC_USER || $bUserForce ? '' : ('sudo ' . ($strUser eq 'root' ? '' : "-u $strUser "))) . $strCommand; # Format and split command $strCommand =~ s/[ ]*\n[ ]*/ \\\n /smg; @@ -170,6 +171,8 @@ sub executeKey output => JSON::PP::false, }; + $$hCacheKey{'run-as-user'} = $bUserForce ? $strUser : undef; + if (defined($oCommand->fieldGet('exe-cmd-extra', false))) { $$hCacheKey{'cmd-extra'} = $oCommand->fieldGet('exe-cmd-extra'); @@ -185,6 +188,9 @@ sub executeKey $$hCacheKey{'output'} = JSON::PP::true; } + $$hCacheKey{'load-env'} = $oCommand->paramTest('load-env', 'n') ? JSON::PP::false : JSON::PP::true; + $$hCacheKey{'bash-wrap'} = $oCommand->paramTest('bash-wrap', 'n') ? JSON::PP::false : JSON::PP::true; + if (defined($oCommand->fieldGet('exe-highlight', false))) { $$hCacheKey{'output'} = JSON::PP::true; @@ -276,7 +282,8 @@ sub execute $strCommand . (defined($$hCacheKey{'cmd-extra'}) ? ' ' . $$hCacheKey{'cmd-extra'} : ''), {iExpectedExitStatus => $$hCacheKey{'err-expect'}, bSuppressError => $oCommand->paramTest('err-suppress', 'y'), - iRetrySeconds => $oCommand->paramGet('retry', false)}); + iRetrySeconds => $oCommand->paramGet('retry', false)}, $hCacheKey->{'run-as-user'}, + {bLoadEnv => $hCacheKey->{'load-env'}, bBashWrap => $hCacheKey->{'bash-wrap'}}); $oExec->begin(); $oExec->end(); diff --git a/doc/xml/dtd/doc.dtd b/doc/xml/dtd/doc.dtd index 57ef72098..3895fedaa 100644 --- a/doc/xml/dtd/doc.dtd +++ b/doc/xml/dtd/doc.dtd @@ -63,6 +63,9 @@ + + + diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 1faf5ec3a..fadc0b71e 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -38,7 +38,11 @@ -

Document generator improvements. Allow parameters to be passed when a container is created. Allow /etc/hosts update to be skipped (for containers without bash).

+

Build containers from scratch for more accurate testing.

+ + + +

Document generator improvements. Allow parameters to be passed when a container is created. Allow /etc/hosts update to be skipped (for containers without bash). Allow environment load to be skipped. Allow bash wrapping to be skipped. Allow forcing a command to run as a user without sudo.

 diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index d27c826bf..789a5c123 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -1,6 +1,6 @@ - + The {[project]} User Guide demonstrates how to quickly and easily setup {[project]} for your {[postgres]} database. Step-by-step instructions lead the user through all the important features of the fastest, most reliable {[postgres]} backup and restore solution. @@ -20,8 +20,10 @@ 9.4 9.5 + 95 9.5 9.6 + 96 /usr/lib/postgresql/{[pg-version]}/bin /usr/pgsql-{[pg-version]}/bin @@ -29,7 +31,8 @@ /usr/lib/postgresql/{[pg-version-upgrade]}/bin /usr/pgsql-{[pg-version-upgrade]}/bin - /home/postgres + /var/lib/postgresql + /var/lib/pgsql /usr/share/perl5 /usr/bin @@ -58,7 +61,7 @@ /var/spool/pgbackrest apt-get install postgresql-{[pg-version-upgrade]} - yum install postgresql96-server + yum install postgresql{[pg-version-upgrade-nodot]}-server /etc/postgresql/{[pg-version]}/{[postgres-cluster-demo]}/postgresql.conf {[pg-path]}/postgresql.conf @@ -86,6 +89,9 @@ u16 co6 + ubuntu:16.04 + centos:6 + use English; getpwuid($UID) eq 'root' ? 'ubuntu' : getpwuid($UID) . '' {[pgbackrest-base-dir]}:/backrest pgbackrest/test @@ -96,19 +102,25 @@ pg1 pg-primary {[host-user]} - {[image-repo]}:{[host-os]}-base + {[host-image]} {[host-mount]} + pg1 + build + root + {[host-image]} + {[host-mount]} + pg2 pg-standby {[host-pg1-user]} - {[host-pg1-image]} + {[host-image]} {[host-mount]} repo1 repository {[host-user]} - {[image-repo]}:{[host-os]}-base + {[host-image]} {[host-mount]} @@ -142,6 +154,30 @@ pg_lsclusters service postgresql-{[pg-version-upgrade]} status + + + + mkdir -p -m 700 /root/.ssh && + echo '-----BEGIN RSA PRIVATE KEY-----' > /root/.ssh/id_rsa && + echo 'MIICXwIBAAKBgQDR0yJsZW5d5LcqteiOtv8d+FFeFFHDPI0VTcTOdMn1iDiIP1ou' >> /root/.ssh/id_rsa && + echo 'X3Q2OyNjsBaDbsRJd+sp9IRq1LKX3zsBcgGZANwm0zduuNEPEU94ajS/uRoejIqY' >> /root/.ssh/id_rsa && + echo '/XkKOpnEF6ZbQ2S7TaE4sWeGLvba7kUFs0QTOO+N+nV2dMbdqZf6C8lazwIDAQAB' >> /root/.ssh/id_rsa && + echo 'AoGBAJXa6xzrnFVmwgK5BKzYuX/YF5TPgk2j80ch0ct50buQXH/Cb0/rUH5i4jWS' >> /root/.ssh/id_rsa && + echo 'T6Hy/DFUehnuzpvV6O9auTOhDs3BhEKFRuRLn1nBwTtZny5Hh+cw7azUCEHFCJlz' >> /root/.ssh/id_rsa && + echo 'makCrVbgawtno6oU/pFgQm1FcxD0f+Me5ruNcLHqUZsPQwkRAkEA+8pG+ckOlz6R' >> /root/.ssh/id_rsa && + echo 'AJLIHedmfcrEY9T7sfdo83bzMOz8H5soUUP4aOTLJYCla1LO7JdDnXMGo0KxaHBP' >> /root/.ssh/id_rsa && + echo 'l8j5zDmVewJBANVVPDJr1w37m0FBi37QgUOAijVfLXgyPMxYp2uc9ddjncif0063' >> /root/.ssh/id_rsa && + echo '0Wc0FQefoPszf3CDrHv/RHvhHq97jXDwTb0CQQDgH83NygoS1r57pCw9chzpG/R0' >> /root/.ssh/id_rsa && + echo 'aMEiSPhCvz757fj+qT3aGIal2AJ7/2c/gRZvwrWNETZ3XIZOUKqIkXzJLPjBAkEA' >> /root/.ssh/id_rsa && + echo 'wnP799W2Y8d4/+VX2pMBkF7lG7sSviHEq1sP2BZtPBRQKSQNvw3scM7XcGh/mxmY' >> /root/.ssh/id_rsa && + echo 'yx0qpqfKa8SKbNgI1+4iXQJBAOlg8MJLwkUtrG+p8wf69oCuZsnyv0K6UMDxm6/8' >> /root/.ssh/id_rsa && + echo 'cbvfmvODulYFaIahaqHWEZoRo5CLYZ7gN43WHPOrKxdDL78=' >> /root/.ssh/id_rsa && + echo '-----END RSA PRIVATE KEY-----' >> /root/.ssh/id_rsa && + echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAAAgQDR0yJsZW5d5LcqteiOtv8d+FFeFFHDPI0VTcTOdMn1iDiIP1ouX3Q2OyNjsBaDbsRJd+sp9IRq1LKX3zsBcgGZANwm0zduuNEPEU94ajS/uRoejIqY/XkKOpnEF6ZbQ2S7TaE4sWeGLvba7kUFs0QTOO+N+nV2dMbdqZf6C8lazw== root@pgbackrest-doc' > /root/.ssh/authorized_keys && + echo 'Host *' > /root/.ssh/config && + echo ' StrictHostKeyChecking no' >> /root/.ssh/config && + chmod 600 /root/.ssh/* + @@ -203,6 +239,146 @@ + + + + Update apt repository and install packages + + + + sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile + + + + + rm /etc/apt/apt.conf.d/70debconf + + + + + apt-get update + + + + + {[ssh-key-install]} + + + + + apt-get install sudo ssh + -y 2>&1 + + + + + service ssh start + + + + + adduser --disabled-password --gecos "" {[host-user]} + + + echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers + + + + + Update yum repository and install packages + + + + {[ssh-key-install]} + + + + + yum install openssh-server openssh-clients sudo + -y 2>&1 + + + + + adduser -n {[host-user]} + + + echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]} + + + + + service sshd start + + + + + + + + + Install <postgres/> + + + apt-get install wget + -y 2>&1 + + + + yum install wget + -y 2>&1 + + + + + echo 'deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main' | + sudo tee -a /etc/apt/sources.list.d/pgdg.list + + + + + + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | + sudo apt-key add - + + + + + + rpm -ivh + http://yum.postgresql.org/9.5/redhat/rhel-6-x86_64/pgdg-centos95-9.5-3.noarch.rpm + + 2>&1 + + + + apt-get update + + + + apt-get install postgresql-common + -y 2>&1 + + + + + sed -i 's/^\#create\_main\_cluster.*$/create\_main\_cluster \= false/' + /etc/postgresql-common/createcluster.conf + + + + + apt-get install postgresql-{[pg-version]} + -y 2>&1 + + + + yum install postgresql{[pg-version-nodot]}-server + -y 2>&1 + + + +

{[user-guide-os]} packages for are available at apt.postgresql.org. If they are not provided for your distribution/version it is easy to download the source and install manually.

@@ -235,7 +411,21 @@

The executable is written in C. This allows certain time-critical commands (like async archive-push/archive-get) to run more quickly.

- Build and Install Binary + Install build dependencies + + + apt-get install build-essential libssl-dev libperl-dev + -y 2>&1 + + + + yum install build-essential gcc openssl-devel perl-ExtUtils-Embed + -y 2>&1 + + + + + Build and install binary make -C /root/pgbackrest-release-{[version]}/src @@ -246,27 +436,17 @@ -

contains embedded Perl which requires some additional modules.

+

contains embedded Perl which requires some additional modules.

- + Install required Perl packages - - apt-get update - - - + apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl -y 2>&1 - -

contains embedded Perl. All required Perl modules are available as standard packages.

- - - Install required Perl packages - - + yum install perl perl-Time-HiRes perl-parent perl-JSON perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL -y 2>&1 @@ -291,7 +471,7 @@ mkdir -p {[backrest-config-include-path]} - touch {[backrest-config-demo]} + touch {[backrest-config-demo]} chmod 640 {[backrest-config-demo]} @@ -411,14 +591,87 @@ Installation - + + + + rm /etc/apt/apt.conf.d/70debconf + + + + + + apt-get update && + apt-get -y install build-essential openssl wget git 2>&1 + + + + + + + mkdir -p -m 755 /etc/fake-cert && + cd /etc/fake-cert && + openssl genrsa -out ca.key 2048 2>&1 && + openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 + -subj "/C=US/ST=Country/L=City/O=Organization/CN=pgbackrest.org" 2>&1 && + openssl genrsa -out server.key 2048 2>&1 && + openssl req -new -key server.key -out server.csr + -subj "/C=US/ST=Country/L=City/O=Organization/CN=*.pgbackrest.org" 2>&1 && + openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 99999 + -sha256 2>&1 && + chmod 644 /etc/fake-cert/* + + + + + + + wget -q -O /root/nodejs.sh https://deb.nodesource.com/setup_6.x && + bash /root/nodejs.sh && + apt-get install -y nodejs + + + + + + + wget -q -O /root/scalitys3.tar.gz https://github.com/scality/S3/archive/GA6.4.2.1.tar.gz && + mkdir /root/scalitys3 && + tar -C /root/scalitys3 --strip-components 1 -xvf /root/scalitys3.tar.gz && + cd /root/scalitys3 && + npm install 2>&1 && + sed -i "0,/,/s//,\n \"certFilePaths\":{\"key\":\"\/etc\/fake\-cert\/server.key\",\"cert\":\"\/etc\/fake\-cert\/server.crt\",\"ca\":\"\/etc\/fake\-cert\/ca.crt\"},/" + ./config.json && + sed -i "s/ort\"\: 8000/ort\"\: 443/" ./config.json + + + + + + + nohup npm start --prefix /root/scalitys3 & + + +

A new host named pg1 is created to contain the demo cluster and run examples.

+ + {[host-pg1]} + +

supports 32-bit distributions that build Perl with 64-bit integer support.

+ + Install Perl + + + yum install perl + -y 2>&1 + + + Check for 64-bit integers @@ -427,6 +680,10 @@ + + {[host-pg1]} + + {[host-pg1]} postgres @@ -438,7 +695,7 @@ Make sure the installation worked - + {[project-exe]} @@ -590,7 +847,7 @@ {[pg-cluster-restart]} - + {[pg-cluster-wait]} @@ -651,7 +908,7 @@ Create the stanza and check the configuration - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create completed successfully @@ -666,7 +923,7 @@ Check the configuration - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info check successfully stored in the archive at @@ -676,7 +933,7 @@ Example of an invalid configuration - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --archive-timeout=.1 check could not find WAL segment|did not reach the archive @@ -692,13 +949,13 @@ Backup the {[postgres-cluster-demo]} cluster - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --log-level-console=info backup no prior backup exists|full backup size - + {[cmd-backup-last]} @@ -710,7 +967,7 @@ Differential backup of the {[postgres-cluster-demo]} cluster - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=diff --log-level-console=info backup diff backup size @@ -748,7 +1005,7 @@ Get info for the {[postgres-cluster-demo]} cluster - + {[project-exe]} info (full|incr|diff) backup @@ -780,7 +1037,7 @@ {[pg-cluster-stop]} - + rm {[pg-path]}/global/pg_control @@ -795,7 +1052,7 @@ could not find the database system - + {[pg-cluster-wait]} @@ -819,7 +1076,7 @@ Remove old files from {[postgres-cluster-demo]} cluster - + find {[pg-path]} -mindepth 1 -delete @@ -827,7 +1084,7 @@ Restore the {[postgres-cluster-demo]} cluster and start <postgres/> - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} restore @@ -835,7 +1092,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -861,7 +1118,7 @@ Incremental backup of the {[postgres-cluster-demo]} cluster with the regularly scheduled checkpoint - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr --log-level-console=info backup backup begins after the next regular checkpoint completes @@ -879,7 +1136,7 @@ Incremental backup of the {[postgres-cluster-demo]} cluster with an immediate checkpoint - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr --log-level-console=info backup backup begins after the requested immediate checkpoint completes @@ -902,7 +1159,7 @@ chmod 550 {[backrest-repo-path]}/backup/{[postgres-cluster-demo]}/ - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr --log-level-console=info backup ERROR: @@ -918,7 +1175,7 @@ chmod 750 {[backrest-repo-path]}/backup/{[postgres-cluster-demo]}/ - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr --log-level-console=info backup ERROR: @@ -938,7 +1195,7 @@ Perform an incremental backup - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr --log-level-console=info backup cluster is already in backup mode|backup begins after the requested immediate checkpoint completes @@ -969,7 +1226,7 @@ Get <backrest/> info in JSON format - + pgbackrest --output=json info @@ -983,22 +1240,22 @@ Load <backrest/> info function for <postgres/> - mkdir -p /home/postgres/pgbackrest/doc/example + mkdir -p {[pg-home-path]}/pgbackrest/doc/example cp -r /backrest/doc/example/* - /home/postgres/pgbackrest/doc/example + {[pg-home-path]}/pgbackrest/doc/example - + cat - /home/postgres/pgbackrest/doc/example/pgsql-pgbackrest-info.sql + {[pg-home-path]}/pgbackrest/doc/example/pgsql-pgbackrest-info.sql - + psql -f - /home/postgres/pgbackrest/doc/example/pgsql-pgbackrest-info.sql + {[pg-home-path]}/pgbackrest/doc/example/pgsql-pgbackrest-info.sql @@ -1007,14 +1264,14 @@ Query last successful backup time and archived WAL - + cat - /home/postgres/pgbackrest/doc/example/pgsql-pgbackrest-query.sql + {[pg-home-path]}/pgbackrest/doc/example/pgsql-pgbackrest-query.sql - + psql -f - /home/postgres/pgbackrest/doc/example/pgsql-pgbackrest-query.sql + {[pg-home-path]}/pgbackrest/doc/example/pgsql-pgbackrest-query.sql @@ -1038,7 +1295,7 @@ Query last successful backup time - + pgbackrest --output=json --stanza=demo info | jq '.[0] | .backup[-1] | .timestamp.stop' @@ -1051,7 +1308,7 @@ Query last archived WAL - + pgbackrest --output=json --stanza=demo info | jq '.[0] | .archive[-1] | .max' @@ -1086,13 +1343,13 @@ Perform a full backup - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=full --log-level-console=detail backup archive retention on backup {[backup-full-first]}|remove archive - + {[cmd-backup-last]} @@ -1102,7 +1359,7 @@ Perform a full backup - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=full --log-level-console=info backup expire full backup set\: {[backup-full-first]}|archive retention on backup {[backup-full-second]}|remove archive @@ -1129,15 +1386,15 @@ Perform differential and incremental backups - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff backup - + {[cmd-backup-last]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=incr backup @@ -1147,7 +1404,7 @@ Perform a differential backup - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff --log-level-console=info backup expire diff backup set: {[backup-diff-second]} @@ -1172,24 +1429,24 @@ Perform differential backup - + {[cmd-backup-last]} - + psql -c " select pg_create_restore_point('generate WAL'); select pg_switch_xlog(); select pg_create_restore_point('generate WAL'); select pg_switch_xlog();" - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff --log-level-console=info backup new backup label - + {[cmd-backup-last]} @@ -1197,7 +1454,7 @@ Expire archive - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --log-level-console=detail --repo1-retention-archive-type=diff --repo1-retention-archive=1 expire archive retention on backup {[backup-diff-first]}|remove archive @@ -1229,7 +1486,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta --log-level-console=detail restore demo\/PG_VERSION - exists and matches backup|check\/clean db path|restore global\/pg_control @@ -1243,7 +1500,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -1260,19 +1517,19 @@ Create two test databases and perform a backup - + psql -c "create database test1;" - + psql -c "create database test2;" - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=incr backup @@ -1282,14 +1539,14 @@ Create a test table in each database - + psql -c "create table test1_table (id int); insert into test1_table (id) values (1);" test1 - + psql -c "create table test2_table (id int); insert into test2_table (id) values (2);" test2 @@ -1302,13 +1559,13 @@ Show space used by test1 database - + psql -Atc "select oid from pg_database where datname = 'test1'" - + du -sh {[pg-path]}/base/{[database-test1-oid]} @@ -1324,7 +1581,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta {[dash]}-db-include=test2 restore @@ -1333,7 +1590,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -1343,7 +1600,7 @@ Demonstrate that the test2 database was recovered - + psql -c "select * from test2_table;" test2 @@ -1355,7 +1612,7 @@ Attempting to connect to the test1 database will produce an error - + psql -c "select * from test1_table;" test1 @@ -1370,7 +1627,7 @@ Show space used by test1 database after recovery - + du -sh {[pg-path]}/base/{[database-test1-oid]} @@ -1382,7 +1639,7 @@ Drop the test1 database - + psql -c "drop database test1;" @@ -1394,7 +1651,7 @@ List remaining databases - + psql -c "select oid, datname from pg_database order by oid;" @@ -1415,11 +1672,11 @@ Backup the {[postgres-cluster-demo]} cluster and create a table with very important data - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --type=diff backup - + psql -c "begin; create table important_table (message text); @@ -1436,7 +1693,7 @@ Get the time from <postgres/> - + psql -Atc "select current_timestamp" @@ -1448,7 +1705,7 @@ Drop the important table - + psql -c "begin; drop table important_table; commit; @@ -1466,7 +1723,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta {[dash]}-type=time "{[dash]}-target={[time-recovery-timestamp]}" restore @@ -1475,7 +1732,7 @@ rm {[postgres-log-demo]} - + cat {[postgres-recovery-demo]} recovery_target_time @@ -1490,11 +1747,11 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} - + psql -c "select * from important_table" {[test-table-data]} @@ -1505,7 +1762,7 @@ Examine the <postgres/> log output - + cat {[postgres-log-demo]} recovery stopping before|last completed transaction|starting point-in-time recovery @@ -1516,7 +1773,7 @@ Drop the important table (again) - + psql -c "begin; drop table important_table; commit; @@ -1530,11 +1787,11 @@ Perform a backup then attempt recovery from that backup - + {[cmd-backup-last]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-type=incr backup @@ -1542,7 +1799,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta {[dash]}-type=time "{[dash]}-target={[time-recovery-timestamp]}" restore @@ -1555,11 +1812,11 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} - + psql -c "select * from important_table" does not exist @@ -1570,7 +1827,7 @@ Examine the <postgres/> log output to discover the recovery was not successful - + cat {[postgres-log-demo]} starting point-in-time recovery|consistent recovery state reached @@ -1581,7 +1838,7 @@ Get backup info for the {[postgres-cluster-demo]} cluster - + {[project-exe]} info {[backup-last]} @@ -1596,7 +1853,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta {[dash]}-type=time "{[dash]}-target={[time-recovery-timestamp]}" @@ -1612,11 +1869,11 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} - + psql -c "select * from important_table" {[test-table-data]} @@ -1627,7 +1884,7 @@ Examine the <postgres/> log output for log messages indicating success - + cat {[postgres-log-demo]} recovery stopping before|last completed transaction|starting point-in-time recovery @@ -1667,17 +1924,52 @@ Create the stanza - + + + apt-get -y install python 2>&1 + + + yum -y install epel-release 2>&1 + + + yum -y install python-pip 2>&1 + + + + wget -q --no-check-certificate -O /root/get-pip.py + https://bootstrap.pypa.io/get-pip.py + + + + -H python /root/get-pip.py + + + pip install {[dash]}-upgrade {[dash]}-no-cache-dir pip==9.0.3 2>&1 + + + -H pip install --upgrade awscli 2>&1 + + + aws configure set region us-east-1 + + + aws configure set aws_access_key_id accessKey1 + + + aws configure set aws_secret_access_key verySecretKey1 + + + echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | sudo tee -a /etc/hosts - + aws s3 --no-verify-ssl mb s3://demo-bucket 2>&1 - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create completed successfully @@ -1688,7 +1980,7 @@ Backup the {[postgres-cluster-demo]} cluster - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} --log-level-console=info backup no prior backup exists|full backup size @@ -1713,7 +2005,7 @@ Stop <backrest/> for the stanza - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stop completed successfully @@ -1722,7 +2014,7 @@ Delete the stanza - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-delete completed successfully @@ -1746,10 +2038,14 @@ + + {[host-repo1]} + +

The {[br-user]} user is created to own the repository. Any user can own the repository but it is best not to use postgres (if it exists) to avoid confusion.

- Create <user>{[br-user]}</user> + Create <user>{[br-user]}</user> user adduser --disabled-password --gecos "" {[br-user]} @@ -1765,12 +2061,12 @@ {[host-repo1]} {[br-user]} - {[br-user]} + {[br-group]} {[host-repo1]} - {[br-group]} + {[br-user]} {[br-group]} @@ -1803,7 +2099,7 @@
Configuration - + Configure the <backrest/> repository path {[backrest-repo-path]} @@ -1811,7 +2107,7 @@

The repository host must be configured with the {[host-pg1]} host/user and database path. The primary will be configured as db1 to allow a standby to be added later.

- + Configure <br-option>pg1-host</br-option>/<br-option>pg1-host-user</br-option> and <br-option>pg1-path</br-option> {[pg-path]} @@ -1857,7 +2153,7 @@ Check the configuration - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} check @@ -1901,7 +2197,7 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore @@ -1909,7 +2205,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -1953,7 +2249,7 @@ - + Configure <backrest/> to use multiple <cmd>backup</cmd> processes 3 @@ -1988,7 +2284,7 @@ Stop the <backrest/> services - + {[project-exe]} stop @@ -2009,7 +2305,7 @@ Stop the <backrest/> services again - + {[project-exe]} stop @@ -2019,7 +2315,7 @@ Start the <backrest/> services - + {[project-exe]} start @@ -2029,7 +2325,7 @@ Stop <backrest/> services for the <id>demo</id> stanza - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} stop @@ -2050,7 +2346,7 @@ Start the <backrest/> services for the <id>demo</id> stanza - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} start @@ -2070,6 +2366,14 @@ + + {[host-pg2]} + + + + {[host-pg2]} + + {[host-pg2]} postgres @@ -2140,15 +2444,15 @@ Restore the {[postgres-cluster-demo]} standby cluster - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} restore - + cat {[postgres-recovery-demo]} @@ -2176,7 +2480,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -2186,7 +2490,7 @@ Examine the <postgres/> log output for log messages indicating success - + cat {[postgres-log-demo]} entering standby mode|database system is ready to accept read only connections @@ -2197,7 +2501,7 @@ Create a new table on the primary - + psql -c " begin; @@ -2215,7 +2519,7 @@ Query new table on the standby - + psql -c "select * from replicated_table;" does not exist @@ -2228,7 +2532,7 @@ Call <code>pg_switch_xlog()</code> - + psql -c "select *, current_timestamp from pg_switch_xlog()"; @@ -2240,7 +2544,7 @@ Now the new table exists on the standby (may require a few retries) - + psql -c " select *, current_timestamp from replicated_table" {[test-table-data]} @@ -2252,7 +2556,7 @@ Check the configuration - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info check all other checks passed @@ -2270,7 +2574,7 @@ Create replication user - + psql -c " create user replicator password 'jw8s0F4' replication"; @@ -2283,7 +2587,7 @@ Create <file>pg_hba.conf</file> entry for replication user - + sh -c 'echo "host replication replicator {[host-pg2-ip]}/32 md5" @@ -2309,7 +2613,7 @@ Configure the replication password in the <file>.pgpass</file> file. - + sh -c 'echo "{[host-pg1-ip]}:*:replication:replicator:jw8s0F4" @@ -2317,7 +2621,7 @@ - + chmod 600 {[postgres-pgpass]} @@ -2331,11 +2635,11 @@ {[pg-cluster-stop]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore - + cat {[postgres-recovery-demo]} @@ -2359,7 +2663,7 @@ {[pg-cluster-start]} - + {[pg-cluster-wait]} @@ -2369,7 +2673,7 @@ Examine the <postgres/> log output for log messages indicating success - + cat {[postgres-log-demo]} started streaming WAL from primary @@ -2380,7 +2684,7 @@ Create a new table on the primary - + psql -c " begin; @@ -2396,7 +2700,7 @@ Query table on the standby - + psql -c " select *, current_timestamp from stream_table" {[test-table-data]} @@ -2462,7 +2766,7 @@ Break streaming replication by changing the replication password - + psql -c "alter user replicator password 'bogus'" @@ -2491,11 +2795,11 @@ Test parallel asynchronous archiving - + rm -f /var/log/pgbackrest/demo-archive-push-async.log - + psql -c " select pg_create_restore_point('test async push'); select pg_switch_xlog(); @@ -2506,7 +2810,7 @@ - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info check WAL segment @@ -2517,7 +2821,7 @@ Check results in the log - + cat /var/log/pgbackrest/demo-archive-push-async.log WAL file\(s\) to archive|pushed WAL file 0000000 @@ -2536,10 +2840,10 @@ Check results in the log - + sleep 5 - + cat /var/log/pgbackrest/demo-archive-get-async.log got WAL file [0-F]{24} from archive @@ -2549,7 +2853,7 @@ Fix streaming replication by changing the replication password - + psql -c "alter user replicator password 'jw8s0F4'" @@ -2563,7 +2867,7 @@

can perform backups on a standby instead of the primary. Standby backups require the {[host-pg2]} host to be configured and the backup-standby option enabled. If more than one standby is configured then the first running standby found will be used for the backup.

- + Configure <br-option>pg2-host</br-option>/<br-option>pg2-host-user</br-option> and <br-option>pg2-path</br-option> {[pg-path]} @@ -2603,14 +2907,18 @@ {[pg-cluster-stop]} + + + rpm -ivh + http://yum.postgresql.org/9.6/redhat/rhel-6-x86_64/pgdg-centos96-9.6-3.noarch.rpm + + 2>&1 + + {[postgres-install-upgrade]} -y - - - pg_dropcluster {[pg-version-upgrade]} main -

Stop the old cluster on the standby since it will be restored from the newly upgraded cluster.

@@ -2622,14 +2930,18 @@ {[pg-cluster-stop]} + + + rpm -ivh + http://yum.postgresql.org/9.6/redhat/rhel-6-x86_64/pgdg-centos96-9.6-3.noarch.rpm + + 2>&1 + + {[postgres-install-upgrade]} -y - - - pg_dropcluster {[pg-version-upgrade]} main -

Create the new cluster and perform upgrade.

@@ -2703,7 +3015,7 @@ {[pg-path-upgrade]} - + Upgrade <br-option>pg1-path</br-option> and <br-option>pg2-path</br-option>, disable backup from standby {[pg-path-upgrade]} @@ -2748,11 +3060,11 @@ Check configuration - + {[pg-cluster-check-upgrade]} - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} check @@ -2818,11 +3130,11 @@ Restore the {[postgres-cluster-demo]} standby cluster - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-delta restore - + {[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} restore @@ -2840,7 +3152,7 @@ {[pg-cluster-start-upgrade]} - + {[pg-cluster-wait]} @@ -2851,7 +3163,7 @@

Backup from standby can be enabled now that the standby is restored.

- + Reenable backup from standby y diff --git a/test/lib/pgBackRestTest/Common/HostGroupTest.pm b/test/lib/pgBackRestTest/Common/HostGroupTest.pm index 3ff708669..dd09bad26 100644 --- a/test/lib/pgBackRestTest/Common/HostGroupTest.pm +++ b/test/lib/pgBackRestTest/Common/HostGroupTest.pm @@ -73,8 +73,8 @@ sub hostAdd if ($bUpdateHosts) { - $oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root'); - $oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root'); + $oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); + $oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); } my $strHostList = $oHost->{strName} . (defined($rstryHostName) ? ' ' . join(' ', @{$rstryHostName}) : ''); @@ -87,12 +87,13 @@ sub hostAdd if ($strOtherHostName ne $oHost->{strName}) { # Add this host IP to all hosts - $oOtherHost->executeSimple("echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root'); + $oOtherHost->executeSimple("echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); # Add all other host IPs to this host if ($bUpdateHosts) { - $oHost->executeSimple("echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root'); + $oHost->executeSimple( + "echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false}); } } } diff --git a/test/lib/pgBackRestTest/Common/HostTest.pm b/test/lib/pgBackRestTest/Common/HostTest.pm index ffd691c99..66e1378ff 100644 --- a/test/lib/pgBackRestTest/Common/HostTest.pm +++ b/test/lib/pgBackRestTest/Common/HostTest.pm @@ -109,14 +109,18 @@ sub execute $strOperation, $strCommand, $oParam, - $strUser + $strUser, + $bLoadEnv, + $bBashWrap, ) = logDebugParam ( __PACKAGE__ . '->execute', \@_, {name => 'strCommand'}, - {name => 'oParam', required=> false}, - {name => 'strUser', required => false} + {name => 'oParam', required => false}, + {name => 'strUser', required => false}, + {name => 'bLoadEnv', optional => true, default => true}, + {name => 'bBashWrap', optional => true, default => true}, ); # Set the user @@ -128,7 +132,8 @@ sub execute $strCommand =~ s/'/'\\''/g; my $oExec = new pgBackRestTest::Common::ExecuteTest( - "docker exec -u ${strUser} $self->{strContainer} bash -l -c '${strCommand}'" , $oParam); + "docker exec -u ${strUser} $self->{strContainer}" . + ($bBashWrap ? " bash" . ($bLoadEnv ? ' -l' : '') . " -c '${strCommand}'" : " ${strCommand}"), $oParam); # Return from function and log return values if any return logDebugReturn @@ -151,17 +156,21 @@ sub executeSimple $strOperation, $strCommand, $oParam, - $strUser + $strUser, + $bLoadEnv, + $bBashWrap, ) = logDebugParam ( __PACKAGE__ . '->executeSimple', \@_, {name => 'strCommand', trace => true}, {name => 'oParam', required=> false, trace => true}, - {name => 'strUser', required => false, trace => true} + {name => 'strUser', required => false, trace => true}, + {name => 'bLoadEnv', optional => true, default => true, trace => true}, + {name => 'bBashWrap', optional => true, default => true}, ); - my $oExec = $self->execute($strCommand, $oParam, $strUser); + my $oExec = $self->execute($strCommand, $oParam, $strUser, {bLoadEnv => $bLoadEnv}); $oExec->begin(); $oExec->end(); diff --git a/test/travis.pl b/test/travis.pl index c16b60869..7ac97c1bc 100755 --- a/test/travis.pl +++ b/test/travis.pl @@ -118,14 +118,6 @@ eval executeTest('sudo apt-get install -y texlive-font-utils latex-xcolor', {bSuppressStdErr => true}); processEnd(); - processBegin(VM_CO6 . ' build'); - executeTest("${strTestExe} --vm-build --vm=" . VM_CO6, {bShowOutputAsync => true}); - processEnd(); - - processBegin(VM_U16 . ' build'); - executeTest("${strTestExe} --vm-build --vm=" . VM_U16, {bShowOutputAsync => true}); - processEnd(); - processBegin('release documentation doc'); executeTest("${strReleaseExe} --build", {bShowOutputAsync => true}); processEnd();