You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-06-27 00:21:08 +02:00
Use minio as local S3 emulator in documentation.
The documentation was relying on a ScalityS3 container built for testing which wasn't very transparent. Instead, use the stock minio container and configure it in the documentation. Also, install certificates and CA so that TLS verification can be enabled.
This commit is contained in:
@ -630,7 +630,6 @@ sub backrestConfig
|
|||||||
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
|
my $oConfigClean = dclone($self->{config}{$strHostName}{$$hCacheKey{file}});
|
||||||
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_LEVEL_STDERR});
|
||||||
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
|
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{&CFGOPT_LOG_TIMESTAMP});
|
||||||
delete($$oConfigClean{&CFGDEF_SECTION_GLOBAL}{'repo1-s3-verify-ssl'});
|
|
||||||
|
|
||||||
if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
|
if (keys(%{$$oConfigClean{&CFGDEF_SECTION_GLOBAL}}) == 0)
|
||||||
{
|
{
|
||||||
@ -1086,7 +1085,7 @@ sub sectionChildProcess
|
|||||||
$self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
|
$self->{oManifest}->variableReplace($oChild->paramGet('user')), $$hCacheKey{os},
|
||||||
defined($oChild->paramGet('mount', false)) ?
|
defined($oChild->paramGet('mount', false)) ?
|
||||||
[$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,
|
[$self->{oManifest}->variableReplace($oChild->paramGet('mount'))] : undef,
|
||||||
$$hCacheKey{option}, $$hCacheKey{param});
|
$$hCacheKey{option}, $$hCacheKey{param}, $$hCacheKey{'update-hosts'});
|
||||||
|
|
||||||
$self->{host}{$$hCacheKey{name}} = $oHost;
|
$self->{host}{$$hCacheKey{name}} = $oHost;
|
||||||
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true);
|
$self->{oManifest}->variableSet('host-' . $hCacheKey->{id} . '-ip', $oHost->{strIP}, true);
|
||||||
@ -1094,7 +1093,7 @@ sub sectionChildProcess
|
|||||||
|
|
||||||
# Add to the host group
|
# Add to the host group
|
||||||
my $oHostGroup = hostGroupGet();
|
my $oHostGroup = hostGroupGet();
|
||||||
$oHostGroup->hostAdd($oHost, {bUpdateHosts => $$hCacheKey{'update-hosts'}});
|
$oHostGroup->hostAdd($oHost);
|
||||||
|
|
||||||
# Execute initialize commands
|
# Execute initialize commands
|
||||||
foreach my $oExecute ($oChild->nodeList('execute', false))
|
foreach my $oExecute ($oChild->nodeList('execute', false))
|
||||||
|
2
doc/resource/fake-cert/.gitignore
vendored
Normal file
2
doc/resource/fake-cert/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
*.csr
|
||||||
|
*.srl
|
28
doc/resource/fake-cert/README.md
Normal file
28
doc/resource/fake-cert/README.md
Normal file
@ -0,0 +1,28 @@
|
|||||||
|
# pgBackRest Documentation Certificates
|
||||||
|
|
||||||
|
The certificates in this directory are used for documentation generation only and should not be used for actual services.
|
||||||
|
|
||||||
|
## pgBackRest CA
|
||||||
|
|
||||||
|
Generate a CA that will be used to sign documentation certificates. It can be installed in the documentation containers to make certificates signed by it valid.
|
||||||
|
|
||||||
|
```
|
||||||
|
cd [pgbackrest-root]/doc/resource/fake-cert
|
||||||
|
|
||||||
|
openssl ecparam -genkey -name prime256v1 | openssl ec -out ca.key
|
||||||
|
openssl req -new -x509 -extensions v3_ca -key ca.key -out ca.crt -days 99999 \
|
||||||
|
-subj "/C=US/ST=All/L=All/O=pgBackRest/CN=pgbackrest.org"
|
||||||
|
```
|
||||||
|
|
||||||
|
## S3 Certificate
|
||||||
|
|
||||||
|
Mimic an S3 certificate for the `us-east-1`/`us-east-2` region to generate S3 documentation.
|
||||||
|
|
||||||
|
```
|
||||||
|
cd [pgbackrest-root]/doc/resource/fake-cert
|
||||||
|
|
||||||
|
openssl ecparam -genkey -name prime256v1 | openssl ec -out s3-server.key
|
||||||
|
openssl req -new -sha256 -nodes -out s3-server.csr -key s3-server.key -config s3.cnf
|
||||||
|
openssl x509 -req -in s3-server.csr -CA ca.crt -CAkey ca.key -CAcreateserial \
|
||||||
|
-out s3-server.crt -days 99999 -extensions v3_req -extfile s3.cnf
|
||||||
|
```
|
13
doc/resource/fake-cert/ca.crt
Normal file
13
doc/resource/fake-cert/ca.crt
Normal file
@ -0,0 +1,13 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIIB+jCCAaCgAwIBAgIJAJDUUhiBUbmEMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT
|
||||||
|
AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr
|
||||||
|
UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDAxOTU5WhgP
|
||||||
|
MjI5MzAzMTAwMDE5NTlaMFcxCzAJBgNVBAYTAlVTMQwwCgYDVQQIDANBbGwxDDAK
|
||||||
|
BgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNrUmVzdDEXMBUGA1UEAwwOcGdiYWNr
|
||||||
|
cmVzdC5vcmcwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQYHUcSknRDL+fgFJZI
|
||||||
|
IC73Ju75yA0203IxPO35i8mVb9CcWVhEgHmS+cQ6SfY6GC7V61VB7gwzQ+XESi2p
|
||||||
|
ndhJo1MwUTAdBgNVHQ4EFgQUYMbKIlTUE6gklw8KcSC6fnlOitwwHwYDVR0jBBgw
|
||||||
|
FoAUYMbKIlTUE6gklw8KcSC6fnlOitwwDwYDVR0TAQH/BAUwAwEB/zAKBggqhkjO
|
||||||
|
PQQDAgNIADBFAiEA1Bzy17/6jQimg3ROZTrVGkRtAuzTtjgDParHFrIhSDoCIH43
|
||||||
|
OeOUaPVb0rXGPLu9rFpjPOmtFSW3lf4skheJMKyN
|
||||||
|
-----END CERTIFICATE-----
|
5
doc/resource/fake-cert/ca.key
Normal file
5
doc/resource/fake-cert/ca.key
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
-----BEGIN EC PRIVATE KEY-----
|
||||||
|
MHcCAQEEIB5f3SxfiZ92GMpuqpfTiPO3xaVOnxRh6qVAoRtu7NOZoAoGCCqGSM49
|
||||||
|
AwEHoUQDQgAEGB1HEpJ0Qy/n4BSWSCAu9ybu+cgNNtNyMTzt+YvJlW/QnFlYRIB5
|
||||||
|
kvnEOkn2Ohgu1etVQe4MM0PlxEotqZ3YSQ==
|
||||||
|
-----END EC PRIVATE KEY-----
|
16
doc/resource/fake-cert/s3-server.crt
Normal file
16
doc/resource/fake-cert/s3-server.crt
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
-----BEGIN CERTIFICATE-----
|
||||||
|
MIICbTCCAhOgAwIBAgIJAODTXyGnxWtVMAoGCCqGSM49BAMCMFcxCzAJBgNVBAYT
|
||||||
|
AlVTMQwwCgYDVQQIDANBbGwxDDAKBgNVBAcMA0FsbDETMBEGA1UECgwKcGdCYWNr
|
||||||
|
UmVzdDEXMBUGA1UEAwwOcGdiYWNrcmVzdC5vcmcwIBcNMTkwNTI3MDIwODEwWhgP
|
||||||
|
MjI5MzAzMTAwMjA4MTBaMIGBMQswCQYDVQQGEwJVUzEMMAoGA1UECAwDQWxsMQww
|
||||||
|
CgYDVQQHDANBbGwxEzARBgNVBAoMCnBnQmFja1Jlc3QxHDAaBgNVBAsME1VuaXQg
|
||||||
|
VGVzdGluZyBEb21haW4xIzAhBgNVBAMMGnMzLnVzLWVhc3QtMS5hbWF6b25hd3Mu
|
||||||
|
Y29tMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEe2dO1v1gE0Qj4H407i0K8tN
|
||||||
|
kASkveckACPFzXs2i/++rZY4bwUub08JcMRv0WWwnRzOoumsN26Ge454vTbjoqOB
|
||||||
|
mjCBlzAJBgNVHRMEAjAAMAsGA1UdDwQEAwIF4DB9BgNVHREEdjB0ghpzMy51cy1l
|
||||||
|
YXN0LTEuYW1hem9uYXdzLmNvbYIcKi5zMy51cy1lYXN0LTEuYW1hem9uYXdzLmNv
|
||||||
|
bYIaczMudXMtZWFzdC0yLmFtYXpvbmF3cy5jb22CHCouczMudXMtZWFzdC0yLmFt
|
||||||
|
YXpvbmF3cy5jb20wCgYIKoZIzj0EAwIDSAAwRQIgLiE7LuK6O/bKo70XPUi6xoDE
|
||||||
|
ew+EHO31klTOeWiS6oMCIQCHMEqSAcDF/gnG/UXnp2viHOFjnY+NZgQo76l+/2mE
|
||||||
|
iQ==
|
||||||
|
-----END CERTIFICATE-----
|
5
doc/resource/fake-cert/s3-server.key
Normal file
5
doc/resource/fake-cert/s3-server.key
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
-----BEGIN EC PRIVATE KEY-----
|
||||||
|
MHcCAQEEIBhweMaCuhrRJy6hLV9X7QRCorDdyiUvSWEySHXZJM4DoAoGCCqGSM49
|
||||||
|
AwEHoUQDQgAEEe2dO1v1gE0Qj4H407i0K8tNkASkveckACPFzXs2i/++rZY4bwUu
|
||||||
|
b08JcMRv0WWwnRzOoumsN26Ge454vTbjog==
|
||||||
|
-----END EC PRIVATE KEY-----
|
25
doc/resource/fake-cert/s3.cnf
Normal file
25
doc/resource/fake-cert/s3.cnf
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
[req]
|
||||||
|
default_bits = 4096
|
||||||
|
prompt = no
|
||||||
|
default_md = sha256
|
||||||
|
req_extensions = v3_req
|
||||||
|
distinguished_name = dn
|
||||||
|
|
||||||
|
[ dn ]
|
||||||
|
C=US
|
||||||
|
ST=All
|
||||||
|
L=All
|
||||||
|
O=pgBackRest
|
||||||
|
OU=Unit Testing Domain
|
||||||
|
CN = s3.us-east-1.amazonaws.com
|
||||||
|
|
||||||
|
[ v3_req ]
|
||||||
|
basicConstraints = CA:FALSE
|
||||||
|
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
|
||||||
|
subjectAltName = @alt_names
|
||||||
|
|
||||||
|
[ alt_names ]
|
||||||
|
DNS.1 = s3.us-east-1.amazonaws.com
|
||||||
|
DNS.2 = *.s3.us-east-1.amazonaws.com
|
||||||
|
DNS.3 = s3.us-east-2.amazonaws.com
|
||||||
|
DNS.4 = *.s3.us-east-2.amazonaws.com
|
@ -200,6 +200,7 @@
|
|||||||
<!ATTLIST host-define from CDATA #REQUIRED>
|
<!ATTLIST host-define from CDATA #REQUIRED>
|
||||||
|
|
||||||
<!ELEMENT host-add (execute*)>
|
<!ELEMENT host-add (execute*)>
|
||||||
|
<!ATTLIST host-add if CDATA "">
|
||||||
<!ATTLIST host-add id CDATA "">
|
<!ATTLIST host-add id CDATA "">
|
||||||
<!ATTLIST host-add name CDATA #REQUIRED>
|
<!ATTLIST host-add name CDATA #REQUIRED>
|
||||||
<!ATTLIST host-add user CDATA #REQUIRED>
|
<!ATTLIST host-add user CDATA #REQUIRED>
|
||||||
|
@ -54,6 +54,14 @@
|
|||||||
|
|
||||||
<variable key="pgbackrest-repo-path">/pgbackrest</variable>
|
<variable key="pgbackrest-repo-path">/pgbackrest</variable>
|
||||||
|
|
||||||
|
<!-- Path where CA certificates are installed -->
|
||||||
|
<variable key="ca-cert-path" if="{[os-type-is-debian]}">/usr/local/share/ca-certificates</variable>
|
||||||
|
<variable key="ca-cert-path" if="{[os-type-is-centos]}">/etc/pki/ca-trust/source/anchors</variable>
|
||||||
|
|
||||||
|
<!-- Path where fake certificates are located -->
|
||||||
|
<variable key="fake-cert-path-relative">resource/fake-cert</variable>
|
||||||
|
<variable key="fake-cert-path">{[pgbackrest-host-repo-path]}/doc/{[fake-cert-path-relative]}</variable>
|
||||||
|
|
||||||
<variable key="pg-version" if="{[os-type-is-debian]}">{[os-debian-pg-version]}</variable>
|
<variable key="pg-version" if="{[os-type-is-debian]}">{[os-debian-pg-version]}</variable>
|
||||||
<variable key="pg-version" if="{[os-type-is-centos6]}">{[os-centos6-pg-version]}</variable>
|
<variable key="pg-version" if="{[os-type-is-centos6]}">{[os-centos6-pg-version]}</variable>
|
||||||
<variable key="pg-version" if="{[os-type-is-centos7]}">{[os-centos7-pg-version]}</variable>
|
<variable key="pg-version" if="{[os-type-is-centos7]}">{[os-centos7-pg-version]}</variable>
|
||||||
@ -128,6 +136,15 @@
|
|||||||
<variable key="pg-switch-wal" if="{[pg-version]} < 10">pg_switch_xlog</variable>
|
<variable key="pg-switch-wal" if="{[pg-version]} < 10">pg_switch_xlog</variable>
|
||||||
<variable key="pg-switch-wal" if="{[pg-version]} >= 10">pg_switch_wal</variable>
|
<variable key="pg-switch-wal" if="{[pg-version]} >= 10">pg_switch_wal</variable>
|
||||||
|
|
||||||
|
<!-- S3 Settings -->
|
||||||
|
<variable key="s3-local">y</variable>
|
||||||
|
<variable key="s3-bucket">demo-bucket</variable>
|
||||||
|
<variable key="s3-repo">demo-repo</variable>
|
||||||
|
<variable key="s3-region">us-east-1</variable>
|
||||||
|
<variable key="s3-endpoint">s3.{[s3-region]}.amazonaws.com</variable>
|
||||||
|
<variable key="s3-key">accessKey1</variable>
|
||||||
|
<variable key="s3-key-secret">verySecretKey1</variable>
|
||||||
|
|
||||||
<!-- Hosts -->
|
<!-- Hosts -->
|
||||||
<variable key="host-image">pgbackrest/doc:{[os-type]}</variable>
|
<variable key="host-image">pgbackrest/doc:{[os-type]}</variable>
|
||||||
|
|
||||||
@ -228,11 +245,18 @@
|
|||||||
echo ' StrictHostKeyChecking no' >> /root/.ssh/config && \
|
echo ' StrictHostKeyChecking no' >> /root/.ssh/config && \
|
||||||
chmod 600 /root/.ssh/*
|
chmod 600 /root/.ssh/*
|
||||||
</variable>
|
</variable>
|
||||||
|
|
||||||
|
<variable key="minio-client-install">wget https://dl.min.io/client/mc/release/linux-amd64/mc -qO /usr/bin/mc && \
|
||||||
|
chmod 755 /usr/bin/mc</variable>
|
||||||
|
|
||||||
|
<variable key="copy-ca-cert">COPY {[fake-cert-path-relative]}/ca.crt {[ca-cert-path]}/pgbackrest-ca.crt</variable>
|
||||||
</variable-list>
|
</variable-list>
|
||||||
|
|
||||||
<!-- Setup hosts used to build the documentation
|
<!-- Setup hosts used to build the documentation
|
||||||
============================================================================================================================ -->
|
============================================================================================================================ -->
|
||||||
<host-define if="{[os-type-is-debian]}" image="{[host-image]}" from="{[os-image]}">
|
<host-define if="{[os-type-is-debian]}" image="{[host-image]}" from="{[os-image]}">
|
||||||
|
{[copy-ca-cert]}
|
||||||
|
|
||||||
# Fix root tty
|
# Fix root tty
|
||||||
RUN sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
|
RUN sed -i 's/^mesg n/tty -s \&\& mesg n/g' /root/.profile && \
|
||||||
|
|
||||||
@ -241,7 +265,10 @@
|
|||||||
|
|
||||||
# Install base packages
|
# Install base packages
|
||||||
RUN apt-get update && \
|
RUN apt-get update && \
|
||||||
apt-get install -y sudo ssh wget vim gnupg lsb-release 2>&1
|
apt-get install -y sudo ssh wget vim gnupg lsb-release iputils-ping ca-certificates 2>&1
|
||||||
|
|
||||||
|
# Install CA certificate
|
||||||
|
RUN update-ca-certificates
|
||||||
|
|
||||||
# Install PostgreSQL
|
# Install PostgreSQL
|
||||||
RUN RELEASE_CODENAME=`lsb_release -c | awk '{print $2}'` && \
|
RUN RELEASE_CODENAME=`lsb_release -c | awk '{print $2}'` && \
|
||||||
@ -261,13 +288,21 @@
|
|||||||
RUN adduser --disabled-password --gecos "" {[host-user]} && \
|
RUN adduser --disabled-password --gecos "" {[host-user]} && \
|
||||||
echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
|
echo '%{[host-user]} ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
|
||||||
|
|
||||||
|
RUN {[minio-client-install]}
|
||||||
|
|
||||||
ENTRYPOINT service ssh restart && bash
|
ENTRYPOINT service ssh restart && bash
|
||||||
</host-define>
|
</host-define>
|
||||||
|
|
||||||
<host-define if="{[os-type-is-centos6]}" image="{[host-image]}" from="{[os-image]}">
|
<host-define if="{[os-type-is-centos6]}" image="{[host-image]}" from="{[os-image]}">
|
||||||
|
{[copy-ca-cert]}
|
||||||
|
|
||||||
# Install packages
|
# Install packages
|
||||||
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1
|
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1
|
||||||
|
|
||||||
|
# Install CA certificate
|
||||||
|
RUN update-ca-trust enable && \
|
||||||
|
update-ca-trust extract
|
||||||
|
|
||||||
# Regenerate SSH keys
|
# Regenerate SSH keys
|
||||||
RUN rm -f /etc/ssh/ssh_host_rsa_key* && \
|
RUN rm -f /etc/ssh/ssh_host_rsa_key* && \
|
||||||
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
|
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key
|
||||||
@ -284,12 +319,16 @@
|
|||||||
RUN adduser -n {[host-user]} && \
|
RUN adduser -n {[host-user]} && \
|
||||||
echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}
|
echo '{[host-user]} ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/{[host-user]}
|
||||||
|
|
||||||
|
RUN {[minio-client-install]}
|
||||||
|
|
||||||
ENTRYPOINT /usr/sbin/sshd -D
|
ENTRYPOINT /usr/sbin/sshd -D
|
||||||
</host-define>
|
</host-define>
|
||||||
|
|
||||||
<host-define if="{[os-type-is-centos7]}" image="{[host-image]}" from="{[os-image]}">
|
<host-define if="{[os-type-is-centos7]}" image="{[host-image]}" from="{[os-image]}">
|
||||||
ENV container docker
|
ENV container docker
|
||||||
|
|
||||||
|
{[copy-ca-cert]}
|
||||||
|
|
||||||
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
|
RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in *; do [ $i == \
|
||||||
systemd-tmpfiles-setup.service ] || rm -f $i; done); \
|
systemd-tmpfiles-setup.service ] || rm -f $i; done); \
|
||||||
rm -f /lib/systemd/system/multi-user.target.wants/*;\
|
rm -f /lib/systemd/system/multi-user.target.wants/*;\
|
||||||
@ -305,6 +344,9 @@
|
|||||||
# Install packages
|
# Install packages
|
||||||
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1
|
RUN yum install -y openssh-server openssh-clients sudo wget vim 2>&1
|
||||||
|
|
||||||
|
# Install CA certificate
|
||||||
|
RUN update-ca-trust extract
|
||||||
|
|
||||||
# Regenerate SSH keys
|
# Regenerate SSH keys
|
||||||
RUN rm -f /etc/ssh/ssh_host_rsa_key* && \
|
RUN rm -f /etc/ssh/ssh_host_rsa_key* && \
|
||||||
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && \
|
ssh-keygen -t rsa -f /etc/ssh/ssh_host_rsa_key && \
|
||||||
@ -328,6 +370,8 @@
|
|||||||
ln -s /usr/lib/systemd/system/systemd-user-sessions.service \
|
ln -s /usr/lib/systemd/system/systemd-user-sessions.service \
|
||||||
/etc/systemd/system/default.target.wants/systemd-user-sessions.service
|
/etc/systemd/system/default.target.wants/systemd-user-sessions.service
|
||||||
|
|
||||||
|
RUN {[minio-client-install]}
|
||||||
|
|
||||||
CMD ["/usr/sbin/init"]
|
CMD ["/usr/sbin/init"]
|
||||||
</host-define>
|
</host-define>
|
||||||
|
|
||||||
@ -544,6 +588,9 @@
|
|||||||
<section id="introduction">
|
<section id="introduction">
|
||||||
<title>Introduction</title>
|
<title>Introduction</title>
|
||||||
|
|
||||||
|
<!-- Create S3 server first to allow it time to boot before being used -->
|
||||||
|
<host-add if="'{[s3-local]}' eq 'y'" id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="minio/minio" os="{[os-type]}" option="-v {[fake-cert-path]}/s3-server.crt:/root/.minio/certs/public.crt:ro -v {[fake-cert-path]}/s3-server.key:/root/.minio/certs/private.key:ro -e MINIO_REGION={[s3-region]} -e MINIO_DOMAIN={[s3-endpoint]} -e MINIO_BROWSER=off -e MINIO_ACCESS_KEY={[s3-key]} -e MINIO_SECRET_KEY={[s3-key-secret]}" param="server /data --address :443 --compat" update-hosts="n"/>
|
||||||
|
|
||||||
<p>This user guide is intended to be followed sequentially from beginning to end &mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
|
<p>This user guide is intended to be followed sequentially from beginning to end &mdash; each section depends on the last. For example, the <link section="/backup">Backup</link> section relies on setup that is performed in the <link section="/quickstart">Quick Start</link> section. Once <backrest/> is up and running then skipping around is possible but following the user guide in order is recommended the first time through.</p>
|
||||||
|
|
||||||
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
|
<p>Although the examples are targeted at {[user-guide-os]} and <postgres/> {[pg-version]}, it should be fairly easy to apply this guide to any Unix distribution and <postgres/> version. The only OS-specific commands are those to create, start, stop, and drop <postgres/> clusters. The <backrest/> commands will be the same on any Unix system though the locations to install Perl libraries and executables may vary.
|
||||||
@ -720,14 +767,6 @@
|
|||||||
<section id="installation">
|
<section id="installation">
|
||||||
<title>Installation</title>
|
<title>Installation</title>
|
||||||
|
|
||||||
<!-- Create S3 server first to allow it time to boot before being used -->
|
|
||||||
<host-add id="{[host-s3-id]}" name="{[host-s3]}" user="root" image="pgbackrest/test:s3-server-20180612A" os="{[os-type]}">
|
|
||||||
<!-- Set host entries to redirect AWS to local s3 server -->
|
|
||||||
<execute user="root" user-force="y">
|
|
||||||
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
|
|
||||||
</execute>
|
|
||||||
</host-add>
|
|
||||||
|
|
||||||
<p>A new host named <host>pg1</host> is created to contain the demo cluster and run <backrest/> examples.</p>
|
<p>A new host named <host>pg1</host> is created to contain the demo cluster and run <backrest/> examples.</p>
|
||||||
|
|
||||||
<host-add id="{[host-pg1-id]}" name="{[host-pg1]}" user="{[host-pg1-user]}" image="{[host-pg1-image]}" os="{[os-type]}" mount="{[host-pg1-mount]}" option="{[host-option]}"/>
|
<host-add id="{[host-pg1-id]}" name="{[host-pg1]}" user="{[host-pg1-user]}" image="{[host-pg1-image]}" os="{[os-type]}" mount="{[host-pg1-mount]}" option="{[host-option]}"/>
|
||||||
@ -1985,11 +2024,20 @@
|
|||||||
|
|
||||||
<p><backrest/> supports locating repositories in <proper>S3-compatible</proper> object stores. The bucket used to store the repository must be created in advance &mdash; <backrest/> will not do it automatically. The repository can be located in the bucket root (<path>/</path>) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.</p>
|
<p><backrest/> supports locating repositories in <proper>S3-compatible</proper> object stores. The bucket used to store the repository must be created in advance &mdash; <backrest/> will not do it automatically. The repository can be located in the bucket root (<path>/</path>) but it's usually best to place it in a subpath so object store logs or other data can also be stored in the bucket without conflicts.</p>
|
||||||
|
|
||||||
<execute-list host="{[host-s3]}" show="n">
|
<execute-list if="'{[s3-local]}' eq 'y'" host="{[host-pg1]}" show="n">
|
||||||
<title>Create the bucket</title>
|
<title>Create the bucket</title>
|
||||||
|
|
||||||
|
<!-- Set host entries to redirect AWS to local s3 server -->
|
||||||
|
<execute user="root" user-force="y" show="n">
|
||||||
|
<exe-cmd>echo "{[host-s3-ip]} {[s3-bucket]}.{[s3-endpoint]} {[s3-endpoint]}" | tee -a /etc/hosts</exe-cmd>
|
||||||
|
</execute>
|
||||||
|
|
||||||
<execute show='n'>
|
<execute show='n'>
|
||||||
<exe-cmd>aws s3 --no-verify-ssl mb s3://demo-bucket 2>&1</exe-cmd>
|
<exe-cmd>mc config host add demo https://{[host-s3-ip]} {[s3-key]} {[s3-key-secret]} --insecure</exe-cmd>
|
||||||
|
</execute>
|
||||||
|
|
||||||
|
<execute show='n'>
|
||||||
|
<exe-cmd>mc mb demo/{[s3-bucket]} --insecure</exe-cmd>
|
||||||
</execute>
|
</execute>
|
||||||
</execute-list>
|
</execute-list>
|
||||||
|
|
||||||
@ -2004,18 +2052,18 @@
|
|||||||
<title>Configure <proper>S3</proper></title>
|
<title>Configure <proper>S3</proper></title>
|
||||||
|
|
||||||
<backrest-config-option section="global" key="repo1-type">s3</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-type">s3</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-path">/demo-repo</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-path">/{[s3-repo]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-key">accessKey1</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-key">{[s3-key]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-key-secret">verySecretKey1</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-key-secret">{[s3-key-secret]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-bucket">demo-bucket</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-bucket">{[s3-bucket]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-endpoint">s3.amazonaws.com</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-endpoint">{[s3-endpoint]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-region">us-east-1</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-region">{[s3-region]}</backrest-config-option>
|
||||||
<backrest-config-option section="global" key="repo1-s3-verify-ssl">n</backrest-config-option>
|
<backrest-config-option section="global" key="repo1-s3-ca-file" if="{[os-type-is-centos]}">/etc/pki/tls/certs/ca-bundle.crt</backrest-config-option>
|
||||||
|
|
||||||
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
|
<backrest-config-option section="global" key="process-max">4</backrest-config-option>
|
||||||
</backrest-config>
|
</backrest-config>
|
||||||
|
|
||||||
<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>us-east-1</id> region.</admonition>
|
<admonition type="note">The region and endpoint will need to be configured to where the bucket is located. The values given here are for the <id>{[s3-region]}</id> region.</admonition>
|
||||||
|
|
||||||
<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>
|
<p>A role should be created to run <backrest/> and the bucket permissions should be set as restrictively as possible. This sample <proper>Amazon S3</proper> policy will restrict all reads and writes to the bucket and repository path.</p>
|
||||||
|
|
||||||
@ -2029,13 +2077,13 @@
|
|||||||
"s3:ListBucket"
|
"s3:ListBucket"
|
||||||
],
|
],
|
||||||
"Resource": [
|
"Resource": [
|
||||||
"arn:aws:s3:::demo-bucket"
|
"arn:aws:s3:::{[s3-bucket]}"
|
||||||
],
|
],
|
||||||
"Condition": {
|
"Condition": {
|
||||||
"StringEquals": {
|
"StringEquals": {
|
||||||
"s3:prefix": [
|
"s3:prefix": [
|
||||||
"",
|
"",
|
||||||
"demo-repo"
|
"{[s3-repo]}"
|
||||||
],
|
],
|
||||||
"s3:delimiter": [
|
"s3:delimiter": [
|
||||||
"/"
|
"/"
|
||||||
@ -2049,12 +2097,12 @@
|
|||||||
"s3:ListBucket"
|
"s3:ListBucket"
|
||||||
],
|
],
|
||||||
"Resource": [
|
"Resource": [
|
||||||
"arn:aws:s3:::demo-bucket"
|
"arn:aws:s3:::{[s3-bucket]}"
|
||||||
],
|
],
|
||||||
"Condition": {
|
"Condition": {
|
||||||
"StringLike": {
|
"StringLike": {
|
||||||
"s3:prefix": [
|
"s3:prefix": [
|
||||||
"demo-repo/*"
|
"{[s3-repo]}/*"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -2067,7 +2115,7 @@
|
|||||||
"s3:DeleteObject"
|
"s3:DeleteObject"
|
||||||
],
|
],
|
||||||
"Resource": [
|
"Resource": [
|
||||||
"arn:aws:s3:::demo-bucket/demo-repo/*"
|
"arn:aws:s3:::{[s3-bucket]}/{[s3-repo]}/*"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
@ -2079,11 +2127,6 @@
|
|||||||
<execute-list host="{[host-pg1]}">
|
<execute-list host="{[host-pg1]}">
|
||||||
<title>Create the stanza</title>
|
<title>Create the stanza</title>
|
||||||
|
|
||||||
<!-- Set host entries to redirect AWS to local s3 server -->
|
|
||||||
<execute user="root" user-force="y" show="n">
|
|
||||||
<exe-cmd>echo "{[host-s3-ip]} demo-bucket.s3.amazonaws.com s3.amazonaws.com" | tee -a /etc/hosts</exe-cmd>
|
|
||||||
</execute>
|
|
||||||
|
|
||||||
<execute user="postgres" output="y">
|
<execute user="postgres" output="y">
|
||||||
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create</exe-cmd>
|
<exe-cmd>{[project-exe]} {[dash]}-stanza={[postgres-cluster-demo]} {[dash]}-log-level-console=info stanza-create</exe-cmd>
|
||||||
<exe-highlight>completed successfully</exe-highlight>
|
<exe-highlight>completed successfully</exe-highlight>
|
||||||
|
16
test/Vagrantfile
vendored
16
test/Vagrantfile
vendored
@ -63,22 +63,6 @@ Vagrant.configure(2) do |config|
|
|||||||
apt-get install -y devscripts build-essential lintian git lcov cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
|
apt-get install -y devscripts build-essential lintian git lcov cloc txt2man debhelper libssl-dev zlib1g-dev libperl-dev \
|
||||||
libxml2-dev liblz4-dev
|
libxml2-dev liblz4-dev
|
||||||
|
|
||||||
#---------------------------------------------------------------------------------------------------------------------------
|
|
||||||
echo 'Install AWS CLI' && date
|
|
||||||
apt-get install -y python-pip
|
|
||||||
pip install --upgrade awscli
|
|
||||||
|
|
||||||
# Configure AWS CLI
|
|
||||||
sudo -i -u vagrant aws configure set region us-east-1
|
|
||||||
sudo -i -u vagrant aws configure set aws_access_key_id accessKey1
|
|
||||||
sudo -i -u vagrant aws configure set aws_secret_access_key verySecretKey1
|
|
||||||
|
|
||||||
# Create test alias for AWS CLI
|
|
||||||
echo '' >> /home/vagrant/.profile
|
|
||||||
echo '# Test alias for AWS CLI' >> /home/vagrant/.profile
|
|
||||||
echo 'alias s3-test="export PYTHONWARNINGS=ignore && aws s3 --endpoint-url=https://172.17.0.2 --no-verify-ssl"' \
|
|
||||||
>> /home/vagrant/.profile
|
|
||||||
|
|
||||||
#---------------------------------------------------------------------------------------------------------------------------
|
#---------------------------------------------------------------------------------------------------------------------------
|
||||||
echo 'Install Devel::Cover' && date
|
echo 'Install Devel::Cover' && date
|
||||||
dpkg -i /backrest/test/package/u18-libdevel-cover-perl_1.29-2_amd64.deb
|
dpkg -i /backrest/test/package/u18-libdevel-cover-perl_1.29-2_amd64.deb
|
||||||
|
@ -59,19 +59,17 @@ sub hostAdd
|
|||||||
$strOperation,
|
$strOperation,
|
||||||
$oHost,
|
$oHost,
|
||||||
$rstryHostName,
|
$rstryHostName,
|
||||||
$bUpdateHosts,
|
|
||||||
) =
|
) =
|
||||||
logDebugParam
|
logDebugParam
|
||||||
(
|
(
|
||||||
__PACKAGE__ . '->hostAdd', \@_,
|
__PACKAGE__ . '->hostAdd', \@_,
|
||||||
{name => 'oHost'},
|
{name => 'oHost'},
|
||||||
{name => 'rstryHostName', optional => true},
|
{name => 'rstryHostName', optional => true},
|
||||||
{name => 'bUpdateHosts', default => true, optional => true},
|
|
||||||
);
|
);
|
||||||
|
|
||||||
$self->{host}{$oHost->{strName}} = $oHost;
|
$self->{host}{$oHost->{strName}} = $oHost;
|
||||||
|
|
||||||
if ($bUpdateHosts)
|
if ($oHost->hostUpdateGet())
|
||||||
{
|
{
|
||||||
$oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
$oHost->executeSimple("echo \"\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
||||||
$oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
$oHost->executeSimple("echo \"# Test Hosts\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
||||||
@ -87,10 +85,14 @@ sub hostAdd
|
|||||||
if ($strOtherHostName ne $oHost->{strName})
|
if ($strOtherHostName ne $oHost->{strName})
|
||||||
{
|
{
|
||||||
# Add this host IP to all hosts
|
# Add this host IP to all hosts
|
||||||
$oOtherHost->executeSimple("echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
if ($oOtherHost->hostUpdateGet())
|
||||||
|
{
|
||||||
|
$oOtherHost->executeSimple(
|
||||||
|
"echo \"$oHost->{strIP} ${strHostList}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
||||||
|
}
|
||||||
|
|
||||||
# Add all other host IPs to this host
|
# Add all other host IPs to this host
|
||||||
if ($bUpdateHosts)
|
if ($oHost->hostUpdateGet())
|
||||||
{
|
{
|
||||||
$oHost->executeSimple(
|
$oHost->executeSimple(
|
||||||
"echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
"echo \"$oOtherHost->{strIP} ${strOtherHostName}\" >> /etc/hosts", undef, 'root', {bLoadEnv => false});
|
||||||
|
@ -41,6 +41,7 @@ sub new
|
|||||||
$self->{stryMount},
|
$self->{stryMount},
|
||||||
$self->{strOption},
|
$self->{strOption},
|
||||||
$self->{strParam},
|
$self->{strParam},
|
||||||
|
$self->{bHostUpdate},
|
||||||
) =
|
) =
|
||||||
logDebugParam
|
logDebugParam
|
||||||
(
|
(
|
||||||
@ -53,6 +54,7 @@ sub new
|
|||||||
{name => 'stryMount', required => false, trace => true},
|
{name => 'stryMount', required => false, trace => true},
|
||||||
{name => 'strOption', required => false, trace => true},
|
{name => 'strOption', required => false, trace => true},
|
||||||
{name => 'strParam', required => false, trace => true},
|
{name => 'strParam', required => false, trace => true},
|
||||||
|
{name => 'bHostUpdate', required => false, trace => true, default => true},
|
||||||
);
|
);
|
||||||
|
|
||||||
executeTest("docker rm -f $self->{strContainer}", {bSuppressError => true});
|
executeTest("docker rm -f $self->{strContainer}", {bSuppressError => true});
|
||||||
@ -250,6 +252,16 @@ sub copyFrom
|
|||||||
return logDebugReturn($strOperation);
|
return logDebugReturn($strOperation);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
####################################################################################################################################
|
||||||
|
# hostUpdateGet
|
||||||
|
####################################################################################################################################
|
||||||
|
sub hostUpdateGet
|
||||||
|
{
|
||||||
|
my $self = shift;
|
||||||
|
|
||||||
|
return $self->{bHostUpdate};
|
||||||
|
}
|
||||||
|
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
# ipGet
|
# ipGet
|
||||||
####################################################################################################################################
|
####################################################################################################################################
|
||||||
|
Reference in New Issue
Block a user