db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Resolve storage path expressions before passing to remote.

Browse Source 
Expressions such as <REPO:ARCHIVE> require a stanza name in order to be resolved correctly.  However, if the stanza name is passed to the remote then that remote will only work correctly for that one stanza.

Instead, resolved the expressions locally but still pass a relative path to the remote.  That way, a storage path that is only configured on the remote does not need to be known locally.
This commit is contained in:
David Steele
2019-02-21 15:40:21 +02:00
parent b4d4680f8c
commit b1eb8af7d5
13 changed files with 81 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/expect/mock-archive-002.log
+24 -24
View File
@@ -13,7 +13,7 @@ P00 INFO: archive-push command end: aborted with exception [055]
> [CONTAINER-EXEC] db-master [BACKREST-BIN] --config=[TEST_PATH]/db-master/pgbackrest.conf --stanza=db archive-get 000000010000000100000001 [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG
------------------------------------------------------------------------------------------------------------------------------------
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=detail --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 ERROR: [055]: unable to load info file '<REPO:ARCHIVE>/archive.info' or '<REPO:ARCHIVE>/archive.info.copy':
P00 ERROR: [055]: unable to load info file 'archive/db/archive.info' or 'archive/db/archive.info.copy':
FileMissingError: raised from remote-1 protocol on 'backup': unable to open '[TEST_PATH]/backup/repo/archive/db/archive.info' for read: [2] No such file or directory
FileMissingError: raised from remote-1 protocol on 'backup': unable to open '[TEST_PATH]/backup/repo/archive/db/archive.info.copy' for read: [2] No such file or directory
HINT: archive.info cannot be opened but is required to push/get WAL segments.
@@ -110,11 +110,11 @@ P00 DEBUG: main::main: => 0
P00 INFO: archive-get command begin [BACKREST-VERSION]: [700000007000000070000000, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 DEBUG: config/load::cfgLoad: => void
P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNew: (path: {"/var/lib/pgbackrest"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNew: (modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: protocol/helper::protocolGet: (remoteType: 0, remoteId: 1)
P00 DEBUG: protocol/helper::protocolParam: (remoteType: 0, remoteId: 1)
P00 DEBUG: protocol/helper::protocolParam: => {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --stanza=db --type=backup remote"]}
P00 DEBUG: common/exec::execNew: (command: {"ssh"}, param: {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --stanza=db --type=backup remote"]}, name: {"remote-1 process on 'backup'"}, timeout: 60000)
P00 DEBUG: protocol/helper::protocolParam: => {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --type=backup remote"]}
P00 DEBUG: common/exec::execNew: (command: {"ssh"}, param: {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --type=backup remote"]}, name: {"remote-1 process on 'backup'"}, timeout: 60000)
P00 DEBUG: common/exec::execNew: => {Exec}
P00 DEBUG: common/exec::execOpen: (this: {Exec})
P00 DEBUG: common/exec::execOpen: => void
@@ -134,15 +134,15 @@ P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_
P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512)
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: <off>}
P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true}
P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false, cipherType: 0)
P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, type: 0, cipherType: 0)
P00 DEBUG: info/info::infoNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, cipherType: 0)
P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false, cipherType: 0)
P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, type: 0, cipherType: 0)
P00 DEBUG: info/info::infoNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, cipherType: 0)
P00 DEBUG: info/info::infoNew: => {Info}
P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg}
P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive}
P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, archiveId: {"9.4-1"}, walSegment: {"700000007000000070000000"})
P00 DEBUG: storage/storage::storageList: (this: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/7000000070000000"}, param.errorOnMissing: false, param.expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: (this: {StorageDriverRemote}, path: {"<REPO:ARCHIVE>/9.4-1/7000000070000000"}, errorOnMissing: false, expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: remote, path: null, write: false}, archiveId: {"9.4-1"}, walSegment: {"700000007000000070000000"})
P00 DEBUG: storage/storage::storageList: (this: {type: remote, path: null, write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/7000000070000000"}, param.errorOnMissing: false, param.expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: (this: {StorageDriverRemote}, path: {"archive/db/9.4-1/7000000070000000"}, errorOnMissing: false, expression: {"^700000007000000070000000-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: => null
P00 DEBUG: storage/storage::storageList: => null
P00 DEBUG: command/archive/common::walSegmentFind: => null
@@ -162,11 +162,11 @@ P00 DEBUG: main::main: => 1
P00 INFO: archive-get command begin [BACKREST-VERSION]: [000000010000000100000001, [TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG] --no-compress --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/db-master/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-console=debug --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --no-log-timestamp --pg1-path=[TEST_PATH]/db-master/db/base --protocol-timeout=60 --repo1-host=backup --repo1-host-cmd=[BACKREST-BIN] --repo1-host-config=[TEST_PATH]/backup/pgbackrest.conf --repo1-host-user=[USER-1] --stanza=db
P00 DEBUG: config/load::cfgLoad: => void
P00 DEBUG: command/archive/get/get::cmdArchiveGet: (void)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNew: (path: {"/var/lib/pgbackrest"}, modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNew: (modeFile: 0640, modePath: 0750, write: false, pathExpressionFunction: (function *))
P00 DEBUG: protocol/helper::protocolGet: (remoteType: 0, remoteId: 1)
P00 DEBUG: protocol/helper::protocolParam: (remoteType: 0, remoteId: 1)
P00 DEBUG: protocol/helper::protocolParam: => {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --stanza=db --type=backup remote"]}
P00 DEBUG: common/exec::execNew: (command: {"ssh"}, param: {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --stanza=db --type=backup remote"]}, name: {"remote-1 process on 'backup'"}, timeout: 60000)
P00 DEBUG: protocol/helper::protocolParam: => {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --type=backup remote"]}
P00 DEBUG: common/exec::execNew: (command: {"ssh"}, param: {["-o", "LogLevel=error", "-o", "Compression=no", "-o", "PasswordAuthentication=no", "pgbackrest@backup", "[BACKREST-BIN] --command=archive-get --compress-level=3 --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/db-master/lock --log-level-file=trace --log-level-stderr=off --log-path=[TEST_PATH]/db-master/log --log-subprocess --pg1-path=[TEST_PATH]/db-master/db/base --process=0 --protocol-timeout=60 --type=backup remote"]}, name: {"remote-1 process on 'backup'"}, timeout: 60000)
P00 DEBUG: common/exec::execNew: => {Exec}
P00 DEBUG: common/exec::execOpen: (this: {Exec})
P00 DEBUG: common/exec::execOpen: => void
@@ -186,15 +186,15 @@ P00 DEBUG: storage/storage::storageNewRead: => {type: posix, name: {"[TEST_
P00 DEBUG: storage/storage::storageGet: (file: {type: posix, name: {"[TEST_PATH]/db-master/db/base/global/pg_control"}, ignoreMissing: false}, param.exactSize: 512)
P00 DEBUG: storage/storage::storageGet: => {used: 512, size: 512, limit: <off>}
P00 DEBUG: postgres/interface::pgControlFromFile: => {version: 90400, systemId: 1000000000000000094, walSegmentSize: 16777216, pageChecksum: true}
P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false, cipherType: 0)
P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, type: 0, cipherType: 0)
P00 DEBUG: info/info::infoNew: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, cipherType: 0)
P00 DEBUG: info/infoArchive::infoArchiveNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, ignoreMissing: false, cipherType: 0)
P00 DEBUG: info/infoPg::infoPgNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, type: 0, cipherType: 0)
P00 DEBUG: info/info::infoNew: (storage: {type: remote, path: null, write: false}, fileName: {"<REPO:ARCHIVE>/archive.info"}, cipherType: 0)
P00 DEBUG: info/info::infoNew: => {Info}
P00 DEBUG: info/infoPg::infoPgNew: => {InfoPg}
P00 DEBUG: info/infoArchive::infoArchiveNew: => {InfoArchive}
P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, archiveId: {"9.4-1"}, walSegment: {"000000010000000100000001"})
P00 DEBUG: storage/storage::storageList: (this: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: (this: {StorageDriverRemote}, path: {"<REPO:ARCHIVE>/9.4-1/0000000100000001"}, errorOnMissing: false, expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: command/archive/common::walSegmentFind: (storage: {type: remote, path: null, write: false}, archiveId: {"9.4-1"}, walSegment: {"000000010000000100000001"})
P00 DEBUG: storage/storage::storageList: (this: {type: remote, path: null, write: false}, pathExp: {"<REPO:ARCHIVE>/9.4-1/0000000100000001"}, param.errorOnMissing: false, param.expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: (this: {StorageDriverRemote}, path: {"archive/db/9.4-1/0000000100000001"}, errorOnMissing: false, expression: {"^000000010000000100000001-[0-f]{40}(\.gz){0,1}$"})
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteList: => {["000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"]}
P00 DEBUG: storage/storage::storageList: => {["000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"]}
P00 DEBUG: command/archive/common::walSegmentFind: => {"000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}
@@ -205,11 +205,11 @@ P00 DEBUG: storage/storage::storageNewWrite: (this: {type: posix, path: {"/
P00 DEBUG: storage/storage::storageNewWrite: => {type: posix, name: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, modeFile: 0640, modePath: 0750, createPath: false, syncFile: false, syncPath: false, atomic: false}
P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: (this: {inputSame: false, done: true}, filter: {IoFilter})
P00 DEBUG: common/io/filter/group::ioFilterGroupAdd: => {inputSame: false, done: true}
P00 DEBUG: storage/storage::storageNewRead: (this: {type: remote, path: {"/var/lib/pgbackrest"}, write: false}, fileExp: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, param.ignoreMissing: false, param.filterGroup: null)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNewRead: (this: {StorageDriverRemote}, file: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNewRead: => {type: remote, name: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}
P00 DEBUG: storage/storage::storageNewRead: => {type: remote, name: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}
P00 DEBUG: storage/storage::storageCopy: (source: {type: remote, name: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}, destination: {type: posix, name: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, modeFile: 0640, modePath: 0750, createPath: false, syncFile: false, syncPath: false, atomic: false})
P00 DEBUG: storage/storage::storageNewRead: (this: {type: remote, path: null, write: false}, fileExp: {"<REPO:ARCHIVE>/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, param.ignoreMissing: false, param.filterGroup: null)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNewRead: (this: {StorageDriverRemote}, file: {"archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false)
P00 DEBUG: storage/driver/remote/storage::storageDriverRemoteNewRead: => {type: remote, name: {"archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}
P00 DEBUG: storage/storage::storageNewRead: => {type: remote, name: {"archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}
P00 DEBUG: storage/storage::storageCopy: (source: {type: remote, name: {"archive/db/9.4-1/0000000100000001/000000010000000100000001-ceb021d9bb41f220511e413b095d2b0d89fec113.gz"}, ignoreMissing: false}, destination: {type: posix, name: {"[TEST_PATH]/db-master/db/base/pg_xlog/RECOVERYXLOG"}, modeFile: 0640, modePath: 0750, createPath: false, syncFile: false, syncPath: false, atomic: false})
P00 DEBUG: storage/storage::storageCopy: => true
P00 DEBUG: command/archive/get/file::archiveGetFile: => 0
P00 INFO: found 000000010000000100000001 in the archive