From 7c95a2f4de43905bc7b22afbc8d70432156bc033 Mon Sep 17 00:00:00 2001 From: David Steele Date: Fri, 8 Sep 2017 21:50:00 -0700 Subject: [PATCH] Updates to installation documentation. * Add trusted SSH configuration. * Add full installation where required and remove doc containers that included parts of the installation. --- doc/xml/release.xml | 8 + doc/xml/user-guide.xml | 574 ++++++++++-------- .../pgBackRestTest/Common/ContainerTest.pm | 45 -- 3 files changed, 324 insertions(+), 303 deletions(-) diff --git a/doc/xml/release.xml b/doc/xml/release.xml index 133702ab9..c3704d6a9 100644 --- a/doc/xml/release.xml +++ b/doc/xml/release.xml @@ -28,6 +28,14 @@

Rename master to primary in documentation to align with convention.

 + + +

Add trusted SSH configuration.

+ + + +

Add full installation where required and remove doc containers that included parts of the installation.

+ diff --git a/doc/xml/user-guide.xml b/doc/xml/user-guide.xml index 33128f17f..8f22dcc63 100644 --- a/doc/xml/user-guide.xml +++ b/doc/xml/user-guide.xml @@ -23,10 +23,16 @@ 9.5 9.6 + /home/postgres + /usr/share/perl5 /usr/bin /var/lib/pgbackrest + backrest + {[br-user]} + /home/{[br-user]} + demo /etc/{[project-exe]}.conf @@ -56,7 +62,7 @@ /etc/postgresql/{[pg-version-upgrade]}/{[postgres-cluster-demo]}/pg_hba.conf {[db-path-upgrade]}/pg_hba.conf - /home/postgres/.pgpass + {[pg-home-path]}/.pgpass /var/log/postgresql/postgresql-{[pg-version]}-{[postgres-cluster-demo]}.log {[db-path]}/pg_log/postgresql.log @@ -78,7 +84,7 @@ db-primary {[host-user]} - {[image-repo]}:{[host-os]}-doc-db + {[image-repo]}:{[host-os]}-base {[host-mount]} db-standby @@ -88,7 +94,7 @@ backup {[host-user]} - {[image-repo]}:{[host-os]}-doc-backup + {[image-repo]}:{[host-os]}-base {[host-mount]} @@ -98,7 +104,7 @@ Important Data - sleep 1 + sleep 2 pg_createcluster {[pg-version]} {[postgres-cluster-demo]} service postgresql-{[pg-version]} initdb @@ -125,6 +131,195 @@ service postgresql-{[pg-version-upgrade]} status + + +

requires trusted (no password) SSH to enable communication between the hosts.

+ + + + + Create <host>{[setup-ssh-host]}</host> host key pair + + + mkdir -m 750 -p {[pg-home-path]}/.ssh + + + ssh-keygen -f {[pg-home-path]}/.ssh/id_rsa -t rsa -b 4096 -N "" + + + +

Exchange keys between {[host-backup]} and {[setup-ssh-host]}.

+ + + Copy <host>{[setup-ssh-host]}</host> public key to <host>{[host-backup]}</host> + + + ssh root@{[setup-ssh-host]} cat {[pg-home-path]}/.ssh/id_rsa.pub | + sudo -u backrest tee -a {[br-home-path]}/.ssh/authorized_keys + + + + + Copy <host>{[host-backup]}</host> public key to <host>{[setup-ssh-host]}</host> + + + ssh root@{[host-backup]} cat {[br-home-path]}/.ssh/id_rsa.pub | + sudo -u postgres tee -a {[pg-home-path]}/.ssh/authorized_keys + + + +

Test that connections can be made from {[host-backup]} to {[setup-ssh-host]} and vice versa.

+ + + Test connection from <host>{[host-backup]}</host> to <host>{[setup-ssh-host]}</host> + + + ssh postgres@{[setup-ssh-host]} + -o StrictHostKeyChecking=no ls + + + + + Test connection from <host>{[setup-ssh-host]}</host> to <host>{[host-backup]}</host> + + + ssh backrest@{[host-backup]} + -o StrictHostKeyChecking=no ls + + + + + + +

is written in Perl which is included with {[user-guide-os]} by default. Some additional modules must also be installed but they are available as standard packages.

+ + + Install required Perl packages + + + apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl + -y 2>&1 + + + +

is written in Perl which is not included with {[user-guide-os]} by default, however all required modules are available as standard packages.

+ + + Install required Perl packages + + + yum install perl perl-Time-HiRes perl-parent perl-JSON + perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL + -y + + + +

{[user-guide-os]} packages for are available at apt.postgresql.org. If they are not provided for your distribution/version it is easy to download the source and install manually.

+ +

{[user-guide-os]} packages for are available from Crunchy Data or yum.postgresql.org, but it is also easy to download the source and install manually.

+ + + Download version <id>{[version]}</id> of <backrest/> + + + + wget -q -O - + {[github-url-release]}/{[version]}.tar.gz | + sudo tar zx -C /root + + + + + mkdir /root/pgbackrest-release-{[version]} + + + cp -r /backrest/bin /root/pgbackrest-release-{[version]} + + + cp -r /backrest/build /root/pgbackrest-release-{[version]} + + + cp -r /backrest/lib /root/pgbackrest-release-{[version]} + + + cp -r /backrest/libc /root/pgbackrest-release-{[version]} + + + cp -r /backrest/src /root/pgbackrest-release-{[version]} + + + + + Install <backrest/> + + + cp -r /root/pgbackrest-release-{[version]}/lib/pgBackRest + {[perl-lib-path]} + + + find {[perl-lib-path]}/pgBackRest -type f -exec chmod 644 {} + + + + find {[perl-lib-path]}/pgBackRest -type d -exec chmod 755 {} + + + + cp /root/pgbackrest-release-{[version]}/bin/{[project-exe]} {[perl-bin-path]}/{[project-exe]} + + + chmod 755 {[perl-bin-path]}/{[project-exe]} + + + mkdir -m 770 /var/log/pgbackrest + + + chown {[br-install-user]}:{[br-install-group]} /var/log/pgbackrest + + + touch {[backrest-config-demo]} + + + chmod 640 {[backrest-config-demo]} + + + chown {[br-install-user]}:{[br-install-group]} {[backrest-config-demo]} + + + + + + + + + + Create the <backrest/> repository + + + mkdir {[backrest-repo-path]} + + + chmod 750 {[backrest-repo-path]} + + + chown {[br-install-user]}:{[br-install-group]} {[backrest-repo-path]} + + + +
Introduction @@ -182,66 +377,11 @@ Installation - - + - - - mkdir /root/pgbackrest-release-{[version]} - - - cp -r /backrest/bin /root/pgbackrest-release-{[version]} - - - cp -r /backrest/build /root/pgbackrest-release-{[version]} - - - cp -r /backrest/lib /root/pgbackrest-release-{[version]} - - - cp -r /backrest/libc /root/pgbackrest-release-{[version]} - - - cp -r /backrest/src /root/pgbackrest-release-{[version]} - - +

A new host named db-primary is created to contain the demo cluster and run examples.

-

is written in Perl which is included with {[user-guide-os]} by default. The DBD::Pg module must also be installed.

- - - Install the <id>DBD::Pg</id> module - - - apt-get install libdbd-pg-perl libio-socket-ssl-perl libxml-libxml-perl - -y 2>&1 - - - -

is written in Perl which is not included with {[user-guide-os]} by default, however all required modules are available as standard packages.

- - - Install required Perl packages - - - yum install perl perl-Time-HiRes perl-parent perl-JSON - perl-Digest-SHA perl-DBD-Pg perl-XML-LibXML perl-IO-Socket-SSL - -y - - - -

{[user-guide-os]} packages for are available, but if they are not provided on your distribution/version it is easy to download the source and install manually.

- -

{[user-guide-os]} packages for are available from Crunchy Data or yum.postgresql.org, but it is also easy to download the source and install manually.

- - - Download version <id>{[version]}</id> of <backrest/> - - - wget -q -O - - {[github-url-release]}/{[version]}.tar.gz | - sudo tar zx -C /root - - +

If has been installed before it's best to be sure that no prior copies of it are still installed. Depending on how old the version of pgBackRest is it may have been installed in a few different locations. The following commands will remove all prior versions of pgBackRest.

@@ -268,61 +408,11 @@ -

The new version can now be installed.

- - - Install <backrest/> - - - cp -r /root/pgbackrest-release-{[version]}/lib/pgBackRest - {[perl-lib-path]} - - - find {[perl-lib-path]}/pgBackRest -type f -exec chmod 644 {} + - - - find {[perl-lib-path]}/pgBackRest -type d -exec chmod 755 {} + - - - cp /root/pgbackrest-release-{[version]}/bin/{[project-exe]} {[perl-bin-path]}/{[project-exe]} - - - chmod 755 {[perl-bin-path]}/{[project-exe]} - - - mkdir -m 770 /var/log/pgbackrest - - - chown postgres:postgres /var/log/pgbackrest - - - touch {[backrest-config-demo]} - - - chmod 640 {[backrest-config-demo]} - - - chown postgres:postgres {[backrest-config-demo]} - - - - - + + {[host-db-primary]} + postgres + postgres +

should now be properly installed but it is best to check. If any dependencies were missed then you will get an error when running from the command line.

@@ -342,7 +432,7 @@

The Quick Start section will cover basic configuration of and and introduce the backup, restore, and info commands.

-
+
Setup Demo Cluster

Creating the demo cluster is optional but is strongly recommended, especially for new users, since the example commands in the user guide reference the demo cluster; the examples assume the demo cluster is running on the default port (i.e. 5432). The cluster will not be started until a later section because there is still some configuration to do.

@@ -419,19 +509,11 @@

For this demonstration the repository will be stored on the same host as the server. This is the simplest configuration and is useful in cases where traditional backup software is employed to backup the database host.

- - Create the <backrest/> repository - - - mkdir {[backrest-repo-path]} - - - chmod 750 {[backrest-repo-path]} - - - chown postgres:postgres {[backrest-repo-path]} - - + + {[host-db-primary]} + postgres + postgres +

The repository path must be configured so knows where to find it.

@@ -1433,60 +1515,68 @@

The configuration described in Quickstart is suitable for simple installations but for enterprise configurations it is more typical to have a dedicated backup host. This separates the backups and WAL archive from the database server so database host failures have less impact. It is still a good idea to employ traditional backup software to backup the backup host.

+
+ Installation + +

A new host named backup is created to store the cluster backups.

+ + + +

The {[br-user]} user is created to own the repository. Any user can own the repository but it is best not to use postgres (if it exists) to avoid confusion.

+ + + Create <user>{[br-user]}</user> + + + adduser --disabled-password --gecos "" {[br-user]} + + + groupadd {[br-group]} + + + adduser -g{[br-group]} -n {[br-user]} + + + + + {[host-backup]} + {[br-user]} + {[br-user]} + + + + {[host-backup]} + {[br-group]} + {[br-group]} + +
+ +
+ Setup Trusted SSH + + + bogus !!! + + + + Create <host>{[host-backup]}</host> host key pair + + + mkdir -m 750 {[br-home-path]}/.ssh + + + ssh-keygen -f {[br-home-path]}/.ssh/id_rsa -t rsa -b 4096 -N "" + + + + + {[host-db-primary]} + +
+ -
- Installation and Configuration - - - - cp -r /backrest/lib/pgBackRest {[perl-lib-path]} - - - find {[perl-lib-path]}/pgBackRest -type f -exec chmod 644 {} + - - - find {[perl-lib-path]}/pgBackRest -type d -exec chmod 755 {} + - - - cp /backrest/bin/{[project-exe]} {[perl-bin-path]}/{[project-exe]} - - - chmod 755 {[perl-bin-path]}/{[project-exe]} - - - mkdir -m 770 /var/log/pgbackrest - - - chown backrest:postgres /var/log/pgbackrest - - - - - +
+ Configuration Configure the <backrest/> repository path @@ -1494,7 +1584,7 @@ {[backrest-repo-path]} -

For this example a new host named backup has been created to store the cluster backups. Follow the instructions in Installation to install , Create the Repository to create the repository and Create the Stanza to create the stanza. The backup host must also be configured with the db-primary host/user and database path. The primary will be configured as db1 to allow a standby to be added later.

+

The backup host must be configured with the db-primary host/user and database path. The primary will be configured as db1 to allow a standby to be added later.

Configure <br-option>db1-host</br-option>/<br-option>db1-user</br-option> and <br-option>db1-path</br-option> @@ -1622,7 +1712,7 @@
-
+
Asynchronous Archiving

The archive-async option offloads WAL archiving to a separate process (or processes) to improve throughput. It works by looking ahead to see which WAL segments are ready to be archived beyond the request that is currently making via the archive_command. WAL segments are transferred to the archive directly from the pg_xlog/pg_wal directory and success is only returned by the archive_command when the WAL segment has been safely stored in the archive.

@@ -1692,7 +1782,7 @@
-
+
Parallel Backup / Restore

offers parallel processing to improve performance of compression and transfer. The number of processes to be used for this feature is set using the --process-max option.

@@ -1745,7 +1835,7 @@
-
+
Starting and Stopping

Sometimes it is useful to prevent from running on a system. For example, when failing over from a primary to a standby it's best to prevent from running on the old primary in case gets restarted or can't be completely killed. This will also prevent from running on cron.

@@ -1827,82 +1917,50 @@

Replication allows multiple copies of a cluster (called standbys) to be created from a single primary. The standbys are useful for balancing reads and to provide redundancy in case the primary host fails.

+ +
+ Installation + +

A new host named db-standby is created to run the standby.

+ + + + + {[host-db-standby]} + postgres + postgres + + +

The demo cluster must be created even though it will be overwritten later.

+ + + Create demo cluster + + + {[db-cluster-create]} + + +
+ + +
+ Setup Trusted SSH + + + bogus !!! + + + + {[host-db-standby]} + +
+
Hot Standby

A hot standby performs replication using the WAL archive and allows read-only queries.

-

A new host named db-standby will be created to run the standby. Follow the instructions in Installation to install and Setup Demo Cluster to setup the demo cluster.

- - - - - apt-get -y install libdbd-pg-perl - - - - yum -y install perl perl-Time-HiRes perl-parent - perl-JSON perl-Digest-SHA perl-DBD-Pg - - - - - cp -r /backrest/lib/pgBackRest {[perl-lib-path]} - - - find {[perl-lib-path]}/pgBackRest -type f -exec chmod 644 {} + - - - find {[perl-lib-path]}/pgBackRest -type d -exec chmod 755 {} + - - - cp /backrest/bin/{[project-exe]} - {[perl-bin-path]}/{[project-exe]} - - - chmod 755 {[perl-bin-path]}/{[project-exe]} - - - mkdir -m 770 /var/log/pgbackrest - - - chown postgres:postgres /var/log/pgbackrest - - - - - - - - {[db-cluster-create]} - - - Set options diff --git a/test/lib/pgBackRestTest/Common/ContainerTest.pm b/test/lib/pgBackRestTest/Common/ContainerTest.pm index e5ae8a7c0..d86e4f662 100755 --- a/test/lib/pgBackRestTest/Common/ContainerTest.pm +++ b/test/lib/pgBackRestTest/Common/ContainerTest.pm @@ -664,51 +664,6 @@ sub containerBuild containerWrite( $oStorageDocker, $strTempPath, $strOS, 'Test', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); } - - # Db doc image - ######################################################################################################################## - if (!$bDeprecated) - { - $strImageParent = containerRepo() . ":${strOS}-base"; - $strImage = "${strOS}-doc-db"; - - $strScript = sshSetup($strOS, POSTGRES_USER, POSTGRES_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER}); - - containerWrite( - $oStorageDocker, $strTempPath, $strOS, "Db Doc", $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); - } - - # Backup doc image - ########################################################################################################################### - if (!$bDeprecated) - { - $strImageParent = containerRepo() . ":${strOS}-base"; - $strImage = "${strOS}-doc-backup"; - $strCopy = undef; - - $strScript .= sectionHeader() . - "# Create pgbackrest user\n" . - ' ' . userCreate($strOS, BACKREST_USER, BACKREST_USER_ID, TEST_GROUP); - - $strScript .= - sshSetup($strOS, BACKREST_USER, TEST_GROUP, $$oVm{$strOS}{&VM_CONTROL_MASTER}); - - #----------------------------------------------------------------------------------------------------------------------- - $strScript .= sectionHeader() . - "# Create pgbackrest.conf\n" . - " touch /etc/pgbackrest.conf && \\\n" . - " chmod 640 /etc/pgbackrest.conf && \\\n" . - " chown " . BACKREST_USER . ":" . POSTGRES_GROUP . " /etc/pgbackrest.conf"; - - $strScript .= sectionHeader() . - "# Setup repository\n" . - " mkdir /var/lib/pgbackrest && \\\n" . - " chown -R " . BACKREST_USER . ":" . POSTGRES_GROUP . " /var/lib/pgbackrest && \\\n" . - " chmod 750 /var/lib/pgbackrest"; - - containerWrite( - $oStorageDocker, $strTempPath, $strOS, 'Backup Doc', $strImageParent, $strImage, $strCopy, $strScript, $bVmForce); - } } &log(INFO, "Build Complete");