db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Use minio as local S3 emulator in documentation.

Browse Source 
The documentation was relying on a ScalityS3 container built for testing which wasn't very transparent.  Instead, use the stock minio container and configure it in the documentation.

Also, install certificates and CA so that TLS verification can be enabled.
This commit is contained in:
David Steele
2019-05-27 07:37:20 -04:00
parent a474ba54c5
commit 3e1b06acaa
13 changed files with 188 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/xml/user-guide.xml
+72 -29
View File
@@ -54,6 +54,14 @@
<variable key="pgbackrest-repo-path">/pgbackrest</variable>
<!-- Path where CA certificates are installed -->
<variable key="ca-cert-path" if="{[os-type-is-debian]}">/usr/local/share/ca-certificates</variable>
<variable key="ca-cert-path" if="{[os-type-is-centos]}">/etc/pki/ca-trust/source/anchors</variable>
<!-- Path where fake certificates are located -->
<variable key="fake-cert-path-relative">resource/fake-cert</variable>
<variable key="fake-cert-path">{[pgbackrest-host-repo-path]}/doc/{[fake-cert-path-relative]}</variable>
<variable key="pg-version" if="{[os-type-is-debian]}">{[os-debian-pg-version]}</variable>
<variable key="pg-version" if="{[os-type-is-centos6]}">{[os-centos6-pg-version]}</variable>
<variable key="pg-version" if="{[os-type-is-centos7]}">{[os-centos7-pg-version]}</variable>
@@ -128,6 +136,15 @@
<variable key="pg-switch-wal" if="{[pg-version]} &lt; 10">pg_switch_xlog</variable>
<variable key="pg-switch-wal" if="{[pg-version]} &gt;= 10">pg_switch_wal</variable>
<!-- S3 Settings -->
<variable key="s3-local">y</variable>
<variable key="s3-bucket">demo-bucket</variable>
<variable key="s3-repo">demo-repo</variable>
<variable key="s3-region">us-east-1</variable>
<variable key="s3-endpoint">s3.{[s3-region]}.amazonaws.com</variable>
<variable key="s3-key">accessKey1</variable>
<variable key="s3-key-secret">verySecretKey1</variable>
<!-- Hosts -->
<variable key="host-image">pgbackrest/doc:{[os-type]}</variable>
@@ -228,11 +245,18 @@
echo ' StrictHostKeyChecking no' >> /root/.ssh/config &amp;&amp; \
chmod 600 /root/.ssh/*
</variable>
<variable key="minio-client-install">wget https://dl.min.io/client/mc/release/linux-amd64/mc -qO /usr/bin/mc &amp;&amp; \
chmod 755 /usr/bin/mc</variable>
<variable key="copy-ca-cert">COPY {[fake-cert-path-relative]}/ca.crt {[ca-cert-path]}/pgbackrest-ca.crt</variable>
</variable-list>
<!-- Setup hosts used to build the documentation
============================================================================================================================ -->
<host-define if="{[os-type-is-debian]}" image="{[host-image]}" from="{[os-image]}">
{[copy-ca-cert]}
# Fix root tty
RUN sed -i 's/^mesg n/tty -s \&amp;\&amp; mesg n/g' /root/.profile &amp;&amp; \
@@ -241,7 +265,10 @@
# Install base packages
RUN apt-get update &amp;&amp; \
apt-get install -y sudo ssh wget vim gnupg lsb-release 2>&amp;1
apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates 2>&amp;1
# Install CA certificate
RUN update-ca-certificates
# Install PostgreSQL
RUN RELEASE_CODENAME=`lsb_release -c | awk '{print $2}'` &amp;&amp; \
@@ -261,13 +288,21 @@
RUN adduser --disabled-password --gecos "" {[host-user]} &amp;&amp; \
echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
RUN {[minio-client-install]}
ENTRYPOINT service ssh restart &amp;&amp; bash
</host-define>
<host-define if="{[os-type-is-centos6]}" image="{[host-image]}" from="{[os-image]}">
{[copy-ca-cert]}
# Install packages
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&amp;1
# Install CA certificate
RUN update-ca-trust enable &amp;&amp; \
update-ca-trust extract
# Regenerate SSH keys
RUN rm -f /etc/ssh/ssh_host_rsa_key* &amp;&amp; \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
@@ -284,12 +319,16 @@
RUN adduser -n {[host-user]} &amp;&amp; \
echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}
RUN {[minio-client-install]}
ENTRYPOINT /usr/sbin/sshd -D
</host-define>
<host-define if="{[os-type-is-centos7]}" image="{[host-image]}" from="{[os-image]}">
ENV container docker
{[copy-ca-cert]}
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
systemd-tmpfiles-setup.service ] || rm -f $i; done); \
rm -f /lib/systemd/system/multi-user.target.wants/*;\
@@ -305,6 +344,9 @@
# Install packages
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&amp;1
# Install CA certificate
RUN update-ca-trust extract
# Regenerate SSH keys
RUN rm -f /etc/ssh/ssh_host_rsa_key* &amp;&amp; \
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key &amp;&amp; \
@@ -328,6 +370,8 @@
ln -s /usr/lib/systemd/system/systemd-user-sessions.service \
/etc/systemd/system/default.target.wants/systemd-user-sessions.service
RUN {[minio-client-install]}
CMD ["/usr/sbin/init"]
</host-define>
@@ -544,6 +588,9 @@
<section id="introduction">
<title>Introduction</title>
<!-- Create S3 server first to allow it time to boot before being used -->
<host-add if="'{[s3-local]}' eq 'y'" id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="minio/minio" os="{[os-type]}" option="-v {[fake-cert-path]}/s3-server.crt:/root/.minio/certs/public.crt:ro -v {[fake-cert-path]}/s3-server.key:/root/.minio/certs/private.key:ro -e MINIO_REGION={[s3-region]} -e MINIO_DOMAIN={[s3-endpoint]} -e MINIO_BROWSER=off -e MINIO_ACCESS_KEY={[s3-key]} -e MINIO_SECRET_KEY={[s3-key-secret]}" param="server /data --address :443 --compat" update-hosts="n"/>
<p>This user guide is intended to be followed sequentially from beginning to end &amp;mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
@@ -720,14 +767,6 @@
<section id="installation">
<title>Installation</title>
<!-- Create S3 server first to allow it time to boot before being used -->
<host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="pgbackrest/test:s3-server-20180612A" os="{[os-type]}">
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>
</host-add>
<p>A new host named <host>pg1</host> is created to contain the demo cluster and run <backrest/> examples.</p>
<host-add id="{[host-pg1-id]}" name="{[host-pg1]}" user="{[host-pg1-user]}" image="{[host-pg1-image]}" os="{[os-type]}" mount="{[host-pg1-mount]}" option="{[host-option]}"/>
@@ -1985,11 +2024,20 @@
<p><backrest/> supports locating repositories in <proper>S3-compatible</proper> object stores. The bucket used to store the repository must be created in advance &amp;mdash; <backrest/> will not do it automatically. The repository can be located in the bucket root (<path>/</path>) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.</p>
<execute-list host="{[host-s3]}" show="n">
<execute-list if="'{[s3-local]}' eq 'y'" host="{[host-pg1]}" show="n">
<title>Create the bucket</title>
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} {[s3-bucket]}.{[s3-endpoint]} {[s3-endpoint]}" | tee -a /etc/hosts</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&amp;1</exe-cmd>
<exe-cmd>mc config host add demo https://{[host-s3-ip]} {[s3-key]} {[s3-key-secret]} --insecure</exe-cmd>
</execute>
<execute show='n'>
<exe-cmd>mc mb demo/{[s3-bucket]} --insecure</exe-cmd>
</execute>
</execute-list>
@@ -2004,18 +2052,18 @@
<title>Configure <proper>S3</proper></title>
<backrest-config-option section="global" key="repo1-type">s3</backrest-config-option>
<backrest-config-option section="global" key="repo1-path">/demo-repo</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key">accessKey1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key-secret">verySecretKey1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-bucket">demo-bucket</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-endpoint">s3.amazonaws.com</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-region">us-east-1</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-verify-ssl">n</backrest-config-option>
<backrest-config-option section="global" key="repo1-path">/{[s3-repo]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key">{[s3-key]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-key-secret">{[s3-key-secret]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-bucket">{[s3-bucket]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-endpoint">{[s3-endpoint]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-region">{[s3-region]}</backrest-config-option>
<backrest-config-option section="global" key="repo1-s3-ca-file" if="{[os-type-is-centos]}">/etc/pki/tls/certs/ca-bundle.crt</backrest-config-option>
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
</backrest-config>
<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>us-east-1</id> region.</admonition>
<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>{[s3-region]}</id> region.</admonition>
<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>
@@ -2029,13 +2077,13 @@
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::demo-bucket"
"arn:aws:s3:::{[s3-bucket]}"
],
"Condition": {
"StringEquals": {
"s3:prefix": [
"",
"demo-repo"
"{[s3-repo]}"
],
"s3:delimiter": [
"/"
@@ -2049,12 +2097,12 @@
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::demo-bucket"
"arn:aws:s3:::{[s3-bucket]}"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"demo-repo/*"
"{[s3-repo]}/*"
]
}
}
@@ -2067,7 +2115,7 @@
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::demo-bucket/demo-repo/*"
"arn:aws:s3:::{[s3-bucket]}/{[s3-repo]}/*"
]
}
]
@@ -2079,11 +2127,6 @@
<execute-list host="{[host-pg1]}">
<title>Create the stanza</title>
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
</execute>
<execute user="postgres" output="y">
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create</exe-cmd>
<exe-highlight>completed successfully</exe-highlight>