db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-06-02 22:57:34 +02:00
Code Issues Releases Activity

Squashed commit of the following:

Browse Source 
commit f0d82151b91828fece28a57d2de24f5ccc92e0d1
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Mar 3 16:38:01 2014 -0500

    Changed thread timeout from WARN to ERROR

commit 571d449717fecee1eac64088bde42606fcddc4eb
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 27 19:15:00 2014 -0500

    Redirect find error output to /dev/null.  Sometimes files are removed from the db while find is running.  We only want to error if the find process errors.

commit 84c4cec257b25fc026f12560d6c0bc5cfb5822ec
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 27 13:00:41 2014 -0500

    Put redirects in common places where errors are be output but we don't want to see them.

commit ae782d08845e2d9f5fd9df8ce8a05cb472da6f56
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 25 07:30:45 2014 -0500

    Made archiving single-threaded so it will work

commit a84478045e7e25d197368a19ac375509f80dfc9c
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 24 18:04:42 2014 -0500

    No attempt to kill threads - just abort

commit da41eb3883622a333e6e84056c05964bc2e4e402
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 24 17:59:42 2014 -0500

    Added more warnings

commit 21b15b7cf6a8b48a4e9c65691eaece5306e47c22
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 24 09:33:47 2014 -0500

    Added thread timeout.

commit 5de8f275f0e9f06a45f1d2dafb540d5db244d505
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 22 11:14:01 2014 -0500

    Removed stderr redirects from the master ssh connection

commit 0e4f0f1308e15dad9e6d6d07cf9bc92bc1aa1492
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 22 10:54:02 2014 -0500

    Fixed issue with archive path expiration

commit e8c7d7aca69dae3ff14cf0065815bced3f2002f1
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 22 09:05:58 2014 -0500

    When archive_retention_type is full, use the oldest backup if there are fewer than required

commit 9270a6789206a0f2fcd5f92d5faaae5e6ac54973
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 21 16:09:34 2014 -0500

    Removed some more stderr redirects

commit 6d975d283073e90247214961e31d49959e7765c6
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 21 08:49:27 2014 -0500

    No longer discarding stderr

commit 0387a8ee09e5fbd66fe8d4dae055f9fea07e6039
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 21 07:34:17 2014 -0500

    replaced process id with thread id.  Added use thread to all modules.

commit ac3ce81621fab54c925eead8c53dee01c797d63e
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 21 06:42:42 2014 -0500

    Add process id to logging, don't generate checksum for 0 length files

commit 3d8e933343d8b7fa0900d37f68cd235b923980bf
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 19 16:06:21 2014 -0500

    More threads

commit 857e1774292da086f3ad4e9fca4ed28efea6683f
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 19 15:13:07 2014 -0500

    Using separate master queues

commit 800cd898317d76b24e8aa7f3acdbcb39cb2de052
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 19 13:37:18 2014 -0500

    Thread files are no longer in an array

commit 81433c0fdde660c3357526a3447cd1146a6b9f8c
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 19 13:03:52 2014 -0500

    Now using one control master for threading.

commit b4ac9552d24a12ddf06d4df293a360cd72a1cf4d
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 18 19:29:57 2014 -0500

    Skipped files removed from manifest, checksums stored

commit 27b820cfb4c5ac0ff5695202cdab98d471e0b95f
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 18 15:42:51 2014 -0500

    Backup handles files that were removed by the db

commit 187fdaf49bd1d9abb1805e28afc2fab9161bc405
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 19:01:06 2014 -0500

    Better compression choices.

commit 5c695a6522fd0092934e60dec7634760cde0673f
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 18:20:55 2014 -0500

    Improved archive logging and put in max limit

commit 13086e96266f8872ad0a07dfbac3289233742242
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 17:02:25 2014 -0500

    Renamed processes.

commit eb25dbdb9dedae5ef7dd35e8e5dbff839168373d
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 16:58:25 2014 -0500

    Added stanza to command string

commit b45d9c946e74434a01e62b371f9c382dcb64210d
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 16:51:05 2014 -0500

    Improved command string

commit 6d7f89fc98ad2e511e6e549943ff5c8311de89cf
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 01:31:24 2014 -0500

    change process string

commit 577c5af33fe05c961273d0d62c42dbd3320b4d87
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 16 01:24:54 2014 -0500

    testing

commit b514058cee5801897530e6d27d1a61b0f6b7692f
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 16:46:13 2014 -0500

    Changed to perl storable - at least it works

commit 392da9162084cb17eb7eff8bf1815e970df8ac3e
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 15:28:22 2014 -0500

    Remove old local archive dirs.

commit 0bc40d206cdbf777f56bfa2cb21472da71185872
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 14:33:31 2014 -0500

    Some fixes for config

commit 0979841f1a08352ece1ade8821d69080f82b1132
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 14:18:15 2014 -0500

    Async compress, thread kill improvements

commit bd00fb7f0ceb86092c3998ac1ac923bb4f28ae2f
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 10:45:01 2014 -0500

    Still working on thread kill

commit 3b2dd68aac9bc00e1b45d2a8d6dfd74da17112f7
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 10:31:57 2014 -0500

    More robust join code.

commit 27bf9b65720e278419c0c0a414141fb7b729c789
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 15 10:09:47 2014 -0500

    First SSH session is shared between objects and thread 0

commit c7aebf05c4e12dc9fbd7ba4bed838c6110e6cdd0
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 14 21:39:18 2014 -0500

    Handle term signals gracefully.

commit d100294894e1f1a5b2d35f6807d3e3a18a8c0b7e
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 14 19:56:28 2014 -0500

    Ability to resume failed backups, better locking

commit 308652cc657f5f015c6affde2d01a784862c96a0
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 14 09:05:14 2014 -0500

    Remove a lot of path creates - percent logging.

commit e14d52c42a48b2c707ae585e9470c637c9966d46
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 14 08:23:25 2014 -0500

    Improved backup threading

commit e93a444a415244de15769f07b69741530254f882
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 23:37:52 2014 -0500

    Changes to manifest.

commit 5c393a0df9beb7ce8e78b3b3627bc1114aa1f94d
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 21:44:19 2014 -0500

    Increased time to wait for log files

commit dccdec2ffa5d145f0dd8efceafaef651f59d72c6
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 21:41:20 2014 -0500

    trying to make labels stick

commit ae3dcc57f01983e4eac6ca72bb482b9fa2e6e4d3
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 21:31:57 2014 -0500

    Thread errors now exit

commit 6d9e7db1c69ee2f862e2941574c5c60ec8d5130f
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 20:54:41 2014 -0500

    Increased thread total.

commit 405d36497dc326b12dbc243a1b3bcd97ce438168
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 20:33:05 2014 -0500

    Less logging

commit a78d677b1e226cf9cda517ffeb662b63480c4099
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 20:29:42 2014 -0500

    Wait for archive files.

commit d4ba078183f0edd290f136cdb13c43f9f70ffd43
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 18:14:15 2014 -0500

    Only save manifest once

commit 1bec36b65b9ed5f2ca31879dc03877708ce048df
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 18:07:58 2014 -0500

    Better thread logging.

commit 2bafa5cf26773ba14da891c1079140c2b9ff08af
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 17:52:38 2014 -0500

    Logging tweak

commit 8453681ccdb41e8ad1e3cf71c2df761bb2341768
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 17:50:14 2014 -0500

    Logging tweaks

commit 4349f0caff1e9a9e1501483af8a64ec594513082
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 17:47:53 2014 -0500

    Fixes to log level and more logging

commit 7d55ce865bf9bf0580c256ae25acbf776692ec3a
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 17:11:53 2014 -0500

    Work on log levels

commit 133e27d813e3e70f7c322fe6705e7104909e8b3f
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 15:26:07 2014 -0500

    Archive tmp file fix.

commit 19f9a5193f547a874b871e7388328043da4ad274
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 14:54:28 2014 -0500

    Minor config changes.

commit 13a1f0cf246f24dd1fba09543b5fc19ebb065c70
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 13:54:43 2014 -0500

    Lots and lots of logging improvements

commit 14b13d48404aedaf629ea647592b058db324b5c4
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 12:02:45 2014 -0500

    Async archive log backup working.

commit f0df759bb97f3499e6c245fee88fceb491e4b551
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 13 00:21:25 2014 -0500

    Small debug fix

commit 7c4d463b1da98c4f9c94693ee64c4776075e36e3
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 23:49:40 2014 -0500

    More balanced backup.

commit b770c8ebece6c40190cc9d62f6798eeb32d4a105
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 22:33:09 2014 -0500

    More work on archive-pull

commit 8e60a3111fd22095c6bdbbe99b51c3af87e661a0
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 20:28:27 2014 -0500

    Lots of archive-pull plumbing

commit 16df430b73834c97410565bd96edeb7e7f79714d
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 17:34:34 2014 -0500

    New constants and basic fork.

commit 564bd65b3e8241113998558208d561c61b73fbb3
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 16:01:48 2014 -0500

    Basic plumbing for archive-pull function

commit 5181282b7402ecbba88544e4cafddac1c7eb8f76
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 08:04:42 2014 -0500

    Command line parameter checks

commit 729b07fed5eaa8247eaee65bd8f3e1a996bd7752
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 07:45:29 2014 -0500

    Hardlink now a conf option.

commit 219120ba811ad102f3c0df60119d645d919d534b
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 12 07:39:42 2014 -0500

    Archiving and flag changes.

commit b9a2e7093ef381ce3b5e6bf2ee9ebf9cf03fa3d7
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 21:17:30 2014 -0500

    Some more file debug info

commit 1a9f73b781db87e98ea1a42db2b43a29dec950f3
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 17:07:49 2014 -0500

    Added timestamp to log

commit 49dbc897a635262db27de5091e8d330f92efa948
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 16:57:51 2014 -0500

    Small file allocation improvements.

commit 9a1de327a197af06ab647d88cd7669ff938b4daa
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 16:37:20 2014 -0500

    Turned ssh compression on

commit 71812fca996ca53d205694c796694bb46dd4fbae
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 16:21:33 2014 -0500

    Thread total is defined in the conf file.

commit 171ca709d62a129907e241012e493162c6be2472
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 16:08:55 2014 -0500

    Better file ordering for threads.

commit c58ccb588e846d7012923a927c2b955cffef3a3c
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 11 15:31:16 2014 -0500

    Parallel backup now working

commit a55b37efc31e6a6074ed98fe8c92ff10cfaf277b
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Feb 7 19:34:14 2014 -0500

    Working on parallel backup

commit 820791cf922e4c186288ee005e87a914df4d6c3e
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 6 17:31:33 2014 -0500

    FF log is skipped for pre-9.3 databases.

commit 60b2be58ac5e410719335993dabf218decc266d5
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 6 16:37:37 2014 -0500

    Finished db object abstraction

commit f20f3b3ee642c02b3b4295b235bc59b99a48af20
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 6 15:54:22 2014 -0500

    Started db object abstraction.

commit 9d08f2a64494c9528c30531bd95b7306484f229e
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Feb 6 12:49:54 2014 -0500

    More robust config.  Retention is read from config.

commit a2b0d7a674a1524f53440be45d333409a3ad1a2b
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 22:26:10 2014 -0500

    Backup expiration working again.  Other changes.

commit 27986e0c10999660078e7c74b25bc99eb3e028ce
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 21:39:08 2014 -0500

    File is now an object.

commit 3e7faf0e5f7d3f61856f0c7f3b50329172e69585
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 16:53:25 2014 -0500

    Some path fixes.

commit 7bee0f3141a301c0a1fa1bb8d092dbbd317fe296
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 15:56:05 2014 -0500

    Moved backup location.

commit a15fcb8c23e11bb2ec056abe13120591cd70c515
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 13:10:36 2014 -0500

    Moved tmp path.

commit 7ffade2453afafb98d2d67d3612742e17075947e
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 11:35:09 2014 -0500

    Fixed file mod time and permissions

commit c86a61ca3e15acc1afc09c400aa866a958e1fd27
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 10:40:49 2014 -0500

    Relative links working for tablespaces.

commit 12a960f91a59fbcce53659b1ed4fbed5a1ba8fc9
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Feb 5 08:21:27 2014 -0500

    File copy supports decompression

commit a81049995859666f378ee804ef15aba1d87543e2
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Feb 4 18:48:39 2014 -0500

    Remote backup fully working.

commit e38c482a26d270b19813c67829f0df977afa0996
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 20:48:02 2014 -0500

    Cleanup

commit 946b8261cfedd62114642344614392426196ba00
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 20:29:03 2014 -0500

    Moved archive_push function

commit d5832a70c05dd40e73d6457e2b287f500454b781
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 20:23:04 2014 -0500

    Moved last backup function.

commit aef6730267af465eea3ab99ecd06917f1628d176
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 19:57:21 2014 -0500

    Moved backup functions.

commit 316a5b559858b3aea89306efd4f0557c2f386d09
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 19:03:17 2014 -0500

    Working on remote

commit fa668d50072542890f15d9ebda77bb2ed552e3e9
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Feb 3 14:50:23 2014 -0500

    Move file init code to file module

commit aef4adf1f3864de9e844d7fb7bb31afce5e887d2
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 2 19:03:05 2014 -0500

    Splitting main perl file

commit 4c2e4f7e4924a0cd4ba9d3cae1f99f5fe115d147
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 2 12:45:59 2014 -0500

    Lots more work on remote functionality

commit bf8ac7abe6210b2300375ba879ade54f8dd69167
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 2 11:12:55 2014 -0500

    manifest_get() now works remotely

commit 4556de472240b161eb3e8ac37ece2eac7f5c3278
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Feb 2 10:40:05 2014 -0500

    Source and destination can now be remote.

commit de972ef5d3bb73f69839a06a43dc9448ccd500cd
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 1 12:57:27 2014 -0500

    More work on file_copy

commit 52ba8801efe419988a8900b5696944e8015a3a12
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Feb 1 11:04:33 2014 -0500

    Working on remote file copy

commit 2ad7a3d815da205d2b84372ad3f8279c2572a76e
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 31 00:04:08 2014 -0500

    Formatting.

commit c5de8fa6e637a877aee003ef623b606d113070c8
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 30 23:58:26 2014 -0500

    Created variable for compression extension

commit 151ff8ba6049b8a4b98900cbbde23bb540d71796
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 30 23:24:34 2014 -0500

    Removed unused strPath param from path_get()

commit 52004d3c84b5886e229dcab67729eff2e1ae7291
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 30 23:16:19 2014 -0500

    Comments

commit 447d83fcdc64c0a075772a6b6f7affb0191e2b2a
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 30 23:09:34 2014 -0500

    Removing more copies.

commit f86618377c5b8efe7fe18683f0fb009f7391ad59
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 30 11:35:34 2014 -0500

    Lots of backup refactoring.

commit e13a706c0a251baf9f69126653bb2eabb78064b5
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 29 15:38:57 2014 -0500

    More moving.  Links now also get references.

commit 54c73b38139127e0acb9a13be6cd7a94d84851ca
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 29 15:03:57 2014 -0500

    Moving stuff around

commit c26b47d4c7d346dd67168333114ee8770baa6efd
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 29 12:06:30 2014 -0500

    Abstracting file functions.

commit 47c3c82540ab3acbf741b6e511b86898e40c9b5c
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 29 06:40:15 2014 -0500

    Fixed checksum function.

commit 12f6d889de8ec6363b3275ccf887b41b3336d994
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Jan 28 17:16:07 2014 -0500

    More modular file copy.

commit 6625ea1405aef8b19b0f5e0c347cb1b7f9761747
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 23 19:39:51 2014 -0500

    Coded paths needed for archive logging

commit cdd1ef73069062720af4061b1955d09c3af27330
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 22 21:40:56 2014 -0500

    Added general path function

commit 161de23c3bf7e4ce7e58f6611acbf227a675cd8f
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 22 19:03:20 2014 -0500

    Starting new structure

commit 89b96954f6f6aa87d7ad138c7a87a8cf64558fc4
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Jan 21 22:02:14 2014 -0500

    Allow any archive file start starts with 24 hex characters.

commit 7e73cf22cf8df6410600760390c7600eff3b5152
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Jan 21 21:59:54 2014 -0500

    Changed where .backup files were copied

commit 62ae46e2139e37d2a036131801c85db13148cd0d
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Jan 21 21:12:24 2014 -0500

    Archive expiration working.

commit a83e94ba27dd0c0958eaca145d4630ef5d7e0fbe
Author: David Steele <github@thelabyrinth.net>
Date:   Tue Jan 21 20:38:58 2014 -0500

    Fixed error in the backup regexp

commit faaaa5690f5c2f7999a692a999e3b3de67ff648f
Author: David Steele <github@thelabyrinth.net>
Date:   Mon Jan 20 20:09:50 2014 -0500

    Working on archive retention.

commit df7ada85c226fb7e851fa4c3e1182e9f1a78fe47
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Jan 18 10:14:35 2014 -0500

    Working on expiration

commit 1e0f2485dbcf73b730fb04e6f2ef212d46df725d
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Jan 18 00:03:32 2014 -0500

    Working on expiration

commit 3719d54cc99d6be4b3d22fb853a2c7ff46b4b3dc
Author: David Steele <github@thelabyrinth.net>
Date:   Wed Jan 15 20:41:29 2014 -0500

    Some notes. removed dead .backup move

commit a7de734105b876654b7cbc8d9d448f2749176369
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Jan 12 20:12:49 2014 -0500

    Some cleanup - fixed tablespace path regression.

commit 05dcbc2246ed949fec256e45c6c441b26f30673b
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Jan 12 15:28:19 2014 -0500

    Working on making backups consistent

commit c1e59048808f969b2380a0607a2447d27822d25b
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Jan 12 01:46:27 2014 -0500

    Fixed xlog copy - working on correctness

commit 3ed83e78f2dd8a424a9adc396aa3c2eeed6cf39f
Author: David Steele <github@thelabyrinth.net>
Date:   Sat Jan 11 02:16:14 2014 -0500

    Working on archive log copy

commit 387439ac8e0b01d61f896c67bd92be88acb9b063
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 16:01:25 2014 -0500

    Working on processing archive logs

commit e23820098d7a1dc83e3e8a1f88616a7aa298e388
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 14:02:56 2014 -0500

    Notices are not emitted on start and stop backup

commit 8b73a26b8425a14ce233f16fa29d38fbaf428331
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 00:49:19 2014 -0500

    Some notes for things to do.

commit 49c1fcd3eb33873d402e486bc2b7ede8280d4fc9
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 00:43:26 2014 -0500

    Backup file is being copied to pg_xlog

commit 8e966c50dd1a28e5cfabc8670c88bfbdcb9913aa
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 00:22:51 2014 -0500

    Removed commented code

commit b5f965942bc370beb7803395f79872323f7c59ba
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 10 00:21:46 2014 -0500

    Added optional hard-linking.

commit cceac444ad357bf8babe36f906bacdc25b845841
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 21:55:09 2014 -0500

    Added comment

commit c47ec3d2609539d5437664aad4535bc2a2edad81
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 21:40:33 2014 -0500

    All backup references are now stored.

commit ed5f8226a4a08ca4665bd49b3413bcaf7e4959c5
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 20:12:46 2014 -0500

    References are from oldest backup

commit bb8fb5b54f14b622af02899c95ad9e8e88d74e8b
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 20:00:34 2014 -0500

    Sparse backup dirs implemented

commit 854972dde65f9560ff8177115e5b2e785e478a28
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 18:23:50 2014 -0500

    Basic deltas work.

commit c89984cd59dc2ca1792d8cd99df1978719aed16e
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 18:02:42 2014 -0500

    Revert 2914a6d..e31b21f

    This rolls back to commit 2914a6dcc67b120be0ee5d808590849ebe9c3664.

commit e31b21f95aa5bad69ab2a31f9fd569d1a2e01a78
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 18:01:34 2014 -0500

    Revert 52d8697..e0be213

    This rolls back to commit 52d86971183da8f8a88775f0196f44c3ab9440a1.

commit e0be2132063932108069a0754d7416c2e2c58624
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 18:01:12 2014 -0500

    Revert 2914a6d..9633e17

    This rolls back to commit 2914a6dcc67b120be0ee5d808590849ebe9c3664.

commit 9633e17d04d00e31c80c3d12f848a5351b97963b
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 17:58:40 2014 -0500

    Revert "Removed tmp file."

    This reverts commit 52d86971183da8f8a88775f0196f44c3ab9440a1.

commit 2914a6dcc67b120be0ee5d808590849ebe9c3664
Author: David Steele <github@thelabyrinth.net>
Date:   Thu Jan 9 17:58:01 2014 -0500

    Working on delta

commit 52d86971183da8f8a88775f0196f44c3ab9440a1
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 3 21:24:13 2014 -0500

    Removed tmp file.

commit ba5075a3e0458874f8b8dff0b4bc1f8cb07243ae
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 3 21:19:41 2014 -0500

    New implementation for archive logging

commit 6379b723dee94d63ef55134468ff86f588c717f8
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 3 20:10:53 2014 -0500

    Set the backup label

commit 78457a2f420428cab317e6cdb57446008ad9751c
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 3 19:59:47 2014 -0500

    Simplified incremental naming

commit ef846ac8db0d44339214676a88e0ee7c06d1a217
Author: David Steele <github@thelabyrinth.net>
Date:   Fri Jan 3 19:28:49 2014 -0500

    Working on backup rotation.

commit e6e995ec19f8cb01311ce0f9f2772edbc009b7d2
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 18 22:08:49 2013 -0500

    Fixed typo in config.

commit ba72eda700cdde982a360cdf99a4a5d367403cf4
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 18 21:30:51 2013 -0500

    Commented out dir mod time variable.

commit 9907c8b7b74bc2b3bef76a47567c695de680a2e3
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Tue Dec 17 07:47:24 2013 -0500

    Set file mtime.

commit b1ffa9419034ef7f3b6795f64633cf3dab87d052
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 16 21:59:22 2013 -0500

    New storage structure, saved archive start/stop location.

commit 6d992855d213edae6f1f6986226c7b9d9ad55116
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sun Dec 15 20:56:42 2013 -0500

    Cleaning up start stop backup code.

commit 0d3fa39117e17d1f7cf83927079da7fdaea826a9
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sun Dec 15 20:04:07 2013 -0500

    Added start and stop backup.

commit 1d21f4f9fa2f659d1f8e441fae00a87d4150373a
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sun Dec 15 18:18:54 2013 -0500

    Added diff to commands.

commit 31fca50eae78b777c1113957806ad8658d744235
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sun Dec 15 17:18:50 2013 -0500

    Backup more or less works - no start/stop backup or archiving.

commit 21b7a3f27a373e877f2da845f6f5ac4974d3daac
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sat Dec 14 15:02:47 2013 -0500

    Added manifest load and save.

commit 8b40ea7d598d3110c04f664b38060336c744290f
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sat Dec 14 14:03:08 2013 -0500

    Tablespace names are now recorded

commit 2ba26ffc262404347a46211545a5a2948a3e656c
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sat Dec 14 13:14:59 2013 -0500

    More refactoring.

commit d12773ca85fb73fedac018aecdc6ea5283ddbd2f
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 11 20:39:07 2013 -0500

    Added generic config command

commit 4aa480b858bba7a0114c292132f7289b726258b0
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 11 20:13:39 2013 -0500

    Cleaned up some var names.

commit 299d402209fd4535f1d7aa8dcb32d6c440f61577
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 11 19:57:54 2013 -0500

    More cleanup

commit aa6e72bb9cf9e0523db135790c2cf3097db4c97d
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 11 19:22:41 2013 -0500

    Refactoring

commit 0dba2ab8c7178ec592fe25a0178b5be482f6d544
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Tue Dec 10 20:16:43 2013 -0500

    Added backup command

commit 0bdccd298269113aa0454506824a230923cfbb17
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Tue Dec 10 19:18:41 2013 -0500

    Reversed order of tests.

commit f09505f60efbd715fd72b617036db4124c2b5054
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Tue Dec 10 19:11:54 2013 -0500

    Working on unit tests for archiving.

commit 39989dd01c2fd77147a1b775ae6b092f3d86c65b
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 9 17:26:47 2013 -0500

    Working on manifest.

commit 02a9a693b8881698008ba9f83cbffcc2e8be7f3f
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Fri Dec 6 20:24:14 2013 -0500

    Working on backup.

commit 4d42d448e95df1c7535c3b73594483462f5f893c
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Thu Dec 5 23:29:05 2013 -0500

    Working on backup.

commit 12518e72967ee6f54419f1acdb8f01b3cb2da26e
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Thu Dec 5 08:59:39 2013 -0500

    Working on backup.

commit ae6886c8e058f3a094d7fbb7544122e3e33dc935
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 4 22:30:26 2013 -0500

    Started backup command.

commit c9c7e340a7d89feb96d1070b45a15dd4fecfeb4d
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Wed Dec 4 21:37:45 2013 -0500

    Added config file.

commit 53b2aae10640d6abb287f9c163f2b6700dc7b9de
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 21:37:01 2013 -0500

    Added note.

commit 4a2b009c2e24c4c7c52849ad53358557658ee2a3
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 21:34:25 2013 -0500

    Removed unused include.

commit 1e114cdc9de43efdf43ccbd1d296c01f297a2907
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 21:26:32 2013 -0500

    Fix comments.

commit 9553baf540404c528b73a1608e5308b93d06d166
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 21:23:10 2013 -0500

    Basic error checking and options for archive-local command

commit c7c049902fd87c6f4c43d25b9d60d9289a1742f5
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 16:21:40 2013 -0500

    Added warnings, comments, removed dead code.

commit 5761832df5ce2e7655602d92f9d1a55a75a75bee
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 15:10:18 2013 -0500

    Implemented strict.

commit a2c98391dfe7f360de7e97f0cb06d55dbb06af3d
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Mon Dec 2 14:34:37 2013 -0500

    Starting on configuration.

commit 2be8c5d9e7cef1256bb9f530d0594aec8bbf1351
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sat Nov 23 20:05:04 2013 -0500

    Working on unit tests.

commit 5d0251be2f05dbb44063b874a62057040dcc0f66
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Sat Nov 23 19:16:09 2013 -0500

    Working on archive-local and beefing up unit tests.

commit 8b37786e5b640c7a1c270388ac1158cce45ebf72
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Fri Nov 22 23:24:37 2013 -0500

    Now connecting to db.

commit 8db52c115a18f690d42db975ee1bb28363b5135f
Author: David Steele <gitlab@thelabyrinth.net>
Date:   Fri Nov 22 17:29:01 2013 -0500

    Fixed whitespace.

commit d86e7eb6276bbe1df025c372690c19da8133c4bb
Author: David Steele <dsteele@Davids-MacBook-Pro.local>
Date:   Wed Nov 20 22:30:26 2013 -0500

    Added checksums.

commit bc46aefe61871e27281b9b8c9ffc185c8e2846af
Author: David Steele <dsteele@Davids-MacBook-Pro.local>
Date:   Wed Nov 20 22:24:30 2013 -0500

    Fixed for OSX.  Do not every use TextEditor on code!

commit e67821a23096f4788f6bef71e7d4d361b7d9858f
Author: David Steele <dsteele@Davids-MacBook-Pro.local>
Date:   Wed Nov 20 21:49:07 2013 -0500

    Working on OSX port.

commit 9ce37308b5a3fb81e204a37cfbe28bec968126e4
Author: David Steele <github@thelabyrinth.net>
Date:   Sun Nov 17 21:48:53 2013 -0500

    Working on archive-local command.

commit 548578c8c96d304fb777c9aeb233733f97145818
Author: David Steele <dsteele@laptop-dev.(none)>
Date:   Sun Nov 17 13:58:21 2013 -0500

    Working on unit test.
This commit is contained in:
David Steele 2014-03-04 08:00:51 -05:00
parent d7283f20f9
commit 80d5332fa0
8 changed files with 3810 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
**/*~
*~

38
pg_backrest.conf Normal file
View File

@ -0,0 +1,38 @@
[global:command]
#compress=pigz --rsyncable --best --stdout %file% # Ubuntu Linux
compress=/usr/bin/gzip --stdout %file%
decompress=/usr/bin/gzip -dc %file%
#checksum=sha1sum %file% | awk '{print $1}' # Ubuntu Linux
checksum=/usr/bin/shasum %file% | awk '{print $1}'
manifest=/opt/local/bin/gfind %path% -printf '%P\t%y\t%u\t%g\t%m\t%T@\t%i\t%s\t%l\n'
psql=/Library/PostgreSQL/9.3/bin/psql -X %option%
[global:log]
level-file=debug
level-console=info
[global:backup]
user=backrest
host=localhost
path=/Users/backrest/test
archive-required=y
thread-max=2
thread-timeout=900
[global:archive]
path=/Users/dsteele/test
compress-async=y
archive-max-mb=500
[global:retention]
full_retention=2
differential_retention=2
archive_retention_type=full
archive_retention=2
[db]
psql_options=--cluster=9.3/main
path=/Users/dsteele/test/db/common
[db:command:option]
psql=--port=6001

514
pg_backrest.pl Executable file
View File

@ -0,0 +1,514 @@
#!/usr/bin/perl
use threads;
use strict;
use warnings;
use File::Basename;
use Getopt::Long;
use Config::IniFiles;
use Carp;
use lib dirname($0);
use pg_backrest_utility;
use pg_backrest_file;
use pg_backrest_backup;
use pg_backrest_db;
# Operation constants
use constant
{
OP_ARCHIVE_PUSH => "archive-push",
OP_ARCHIVE_PULL => "archive-pull",
OP_BACKUP => "backup",
OP_EXPIRE => "expire",
};
use constant
{
CONFIG_SECTION_COMMAND => "command",
CONFIG_SECTION_COMMAND_OPTION => "command:option",
CONFIG_SECTION_LOG => "log",
CONFIG_SECTION_BACKUP => "backup",
CONFIG_SECTION_ARCHIVE => "archive",
CONFIG_SECTION_RETENTION => "retention",
CONFIG_SECTION_STANZA => "stanza",
CONFIG_KEY_USER => "user",
CONFIG_KEY_HOST => "host",
CONFIG_KEY_PATH => "path",
CONFIG_KEY_THREAD_MAX => "thread-max",
CONFIG_KEY_THREAD_TIMEOUT => "thread-timeout",
CONFIG_KEY_HARDLINK => "hardlink",
CONFIG_KEY_ARCHIVE_REQUIRED => "archive-required",
CONFIG_KEY_ARCHIVE_MAX_MB => "archive-max-mb",
CONFIG_KEY_LEVEL_FILE => "level-file",
CONFIG_KEY_LEVEL_CONSOLE => "level-console",
CONFIG_KEY_COMPRESS => "compress",
CONFIG_KEY_COMPRESS_ASYNC => "compress-async",
CONFIG_KEY_DECOMPRESS => "decompress",
CONFIG_KEY_CHECKSUM => "checksum",
CONFIG_KEY_MANIFEST => "manifest",
CONFIG_KEY_PSQL => "psql"
};
# Command line parameters
my $strConfigFile; # Configuration file
my $strStanza; # Stanza in the configuration file to load
my $strType; # Type of backup: full, differential (diff), incremental (incr)
GetOptions ("config=s" => \$strConfigFile,
"stanza=s" => \$strStanza,
"type=s" => \$strType)
or die("Error in command line arguments\n");
# Global variables
my %oConfig;
####################################################################################################################################
# CONFIG_LOAD - Get a value from the config and be sure that it is defined (unless bRequired is false)
####################################################################################################################################
sub config_load
{
my $strSection = shift;
my $strKey = shift;
my $bRequired = shift;
my $strDefault = shift;
# Default is that the key is not required
if (!defined($bRequired))
{
$bRequired = false;
}
my $strValue;
# Look in the default stanza section
if ($strSection eq CONFIG_SECTION_STANZA)
{
$strValue = $oConfig{"${strStanza}"}{"${strKey}"};
}
# Else look in the supplied section
else
{
# First check the stanza section
$strValue = $oConfig{"${strStanza}:${strSection}"}{"${strKey}"};
# If the stanza section value is undefined then check global
if (!defined($strValue))
{
$strValue = $oConfig{"global:${strSection}"}{"${strKey}"};
}
}
if (!defined($strValue) && $bRequired)
{
if (defined($strDefault))
{
return $strDefault;
}
confess &log(ERROR, "config value " . (defined($strSection) ? $strSection : "[stanza]") . "->${strKey} is undefined");
}
if ($strSection eq CONFIG_SECTION_COMMAND)
{
my $strOption = config_load(CONFIG_SECTION_COMMAND_OPTION, $strKey);
if (defined($strOption))
{
$strValue =~ s/\%option\%/${strOption}/g;
}
}
return $strValue;
}
####################################################################################################################################
# SAFE_EXIT - terminate all SSH sessions when the script is terminated
####################################################################################################################################
sub safe_exit
{
my $iTotal = backup_thread_kill();
confess &log(ERROR, "process was terminated on signal, ${iTotal} threads stopped");
}
$SIG{TERM} = \&safe_exit;
$SIG{HUP} = \&safe_exit;
$SIG{INT} = \&safe_exit;
####################################################################################################################################
# START MAIN
####################################################################################################################################
# Get the operation
my $strOperation = $ARGV[0];
# Validate the operation
if (!defined($strOperation))
{
confess &log(ERROR, "operation is not defined");
}
if ($strOperation ne OP_ARCHIVE_PUSH &&
$strOperation ne OP_ARCHIVE_PULL &&
$strOperation ne OP_BACKUP &&
$strOperation ne OP_EXPIRE)
{
confess &log(ERROR, "invalid operation ${strOperation}");
}
# Type should only be specified for backups
if (defined($strType) && $strOperation ne OP_BACKUP)
{
confess &log(ERROR, "type can only be specified for the backup operation")
}
####################################################################################################################################
# LOAD CONFIG FILE
####################################################################################################################################
if (!defined($strConfigFile))
{
$strConfigFile = "/etc/pg_backrest.conf";
}
tie %oConfig, 'Config::IniFiles', (-file => $strConfigFile) or confess &log(ERROR, "unable to find config file ${strConfigFile}");
# Load and check the cluster
if (!defined($strStanza))
{
confess "a backup stanza must be specified - show usage";
}
# Set the log levels
log_level_set(uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_FILE, true, "INFO")),
uc(config_load(CONFIG_SECTION_LOG, CONFIG_KEY_LEVEL_CONSOLE, true, "ERROR")));
####################################################################################################################################
# ARCHIVE-PUSH Command
####################################################################################################################################
if ($strOperation eq OP_ARCHIVE_PUSH || $strOperation eq OP_ARCHIVE_PULL)
{
# If an archive section has been defined, use that instead of the backup section when operation is OP_ARCHIVE_PUSH
my $strSection = defined(config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH)) ? CONFIG_SECTION_ARCHIVE : CONFIG_SECTION_BACKUP;
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $bCompressAsync = config_load($strSection, CONFIG_KEY_COMPRESS_ASYNC, true, "n") eq "n" ? false : true;
# Get the async compress flag. If compress_async=y then compression is off for the initial push
my $strStopFile;
my $strArchivePath;
# If logging locally then create the stop archiving file name
if ($strSection eq CONFIG_SECTION_ARCHIVE)
{
$strArchivePath = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_PATH);
$strStopFile = "${strArchivePath}/lock/${strStanza}-archive.stop";
}
# Perform the archive-push
if ($strOperation eq OP_ARCHIVE_PUSH)
{
# Call the archive_push function
if (!defined($ARGV[1]))
{
confess &log(ERROR, "source archive file not provided - show usage");
}
# If the stop file exists then discard the archive log
if (defined($strStopFile))
{
if (-e $strStopFile)
{
&log(ERROR, "archive stop file exists ($strStopFile), discarding " . basename($ARGV[1]));
exit 0;
}
}
# Make sure that archive-push is running locally
if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
{
confess &log(ERROR, "stanza host cannot be set on archive-push - must be run locally on db server");
}
# Get the compress flag
my $bCompress = $bCompressAsync ? false : config_load($strSection, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
# Get the checksum flag
my $bChecksum = config_load($strSection, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - this is the minimal config needed to run archiving
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load($strSection, CONFIG_KEY_USER),
strBackupHost => config_load($strSection, CONFIG_KEY_HOST),
strBackupPath => config_load($strSection, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum
);
&log(INFO, "pushing archive log " . $ARGV[1] . ($bCompressAsync ? " asynchronously" : ""));
archive_push($ARGV[1]);
# Only continue if we are archiving local and a backup server is defined
if (!($strSection eq CONFIG_SECTION_ARCHIVE && defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST))))
{
exit 0;
}
# Set the operation so that archive-pull will be called next
$strOperation = OP_ARCHIVE_PULL;
# fork and exit the parent process
if (fork())
{
exit 0;
}
}
# Perform the archive-pull
if ($strOperation eq OP_ARCHIVE_PULL)
{
# Make sure that archive-pull is running on the db server
if (defined(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST)))
{
confess &log(ERROR, "stanza host cannot be set on archive-pull - must be run locally on db server");
}
# Create a lock file to make sure archive-pull does not run more than once
my $strLockFile = "${strArchivePath}/lock/${strStanza}-archive.lock";
if (!lock_file_create($strLockFile))
{
&log(DEBUG, "archive-pull process is already running - exiting");
exit 0
}
# Build the basic command string that will be used to modify the command during processing
my $strCommand = $^X . " " . $0 . " --stanza=${strStanza}";
# Get the new operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
my $iArchiveMaxMB = config_load(CONFIG_SECTION_ARCHIVE, CONFIG_KEY_ARCHIVE_MAX_MB);
eval
{
# Run file_init_archive - this is the minimal config needed to run archive pulling
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
undef,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
# Call the archive_pull function Continue to loop as long as there are files to process.
while (archive_pull($strArchivePath . "/archive/${strStanza}", $strStopFile, $strCommand, $iArchiveMaxMB))
{
}
};
# If there were errors above then start compressing
if ($@)
{
if ($bCompressAsync)
{
&log(ERROR, "error during transfer: $@");
&log(WARN, "errors during transfer, starting compression");
# Run file_init_archive - this is the minimal config needed to run archive pulling !!! need to close the old file
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => false,
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST)
);
backup_init
(
undef,
$oFile,
undef,
undef,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
undef,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
archive_compress($strArchivePath . "/archive/${strStanza}", $strCommand, 256);
}
else
{
confess $@;
}
}
lock_file_remove();
}
exit 0;
}
####################################################################################################################################
# OPEN THE LOG FILE
####################################################################################################################################
if (defined(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST)))
{
confess &log(ASSERT, "backup/expire operations must be performed locally on the backup server");
}
log_file_set(config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true) . "/log/${strStanza}");
####################################################################################################################################
# GET MORE CONFIG INFO
####################################################################################################################################
# Set the backup type
if (!defined($strType))
{
$strType = "incremental";
}
elsif ($strType eq "diff")
{
$strType = "differential";
}
elsif ($strType eq "incr")
{
$strType = "incremental";
}
elsif ($strType ne "full" && $strType ne "differential" && $strType ne "incremental")
{
confess &log(ERROR, "backup type must be full, differential (diff), incremental (incr)");
}
# Get the operational flags
my $bCompress = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_COMPRESS, true, "y") eq "y" ? true : false;
my $bChecksum = config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_CHECKSUM, true, "y") eq "y" ? true : false;
# Run file_init_archive - the rest of the file config required for backup and restore
my $oFile = pg_backrest_file->new
(
strStanza => $strStanza,
bNoCompression => !$bCompress,
strBackupUser => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_USER),
strBackupHost => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HOST),
strBackupPath => config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_PATH, true),
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandChecksum => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_CHECKSUM, $bChecksum),
strCommandCompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_COMPRESS, $bCompress),
strCommandDecompress => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_DECOMPRESS, $bCompress),
strCommandManifest => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_MANIFEST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL)
);
my $oDb = pg_backrest_db->new
(
strDbUser => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_USER),
strDbHost => config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_HOST),
strCommandPsql => config_load(CONFIG_SECTION_COMMAND, CONFIG_KEY_PSQL),
oDbSSH => $oFile->{oDbSSH}
);
# Run backup_init - parameters required for backup and restore operations
backup_init
(
$oDb,
$oFile,
$strType,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_HARDLINK, true, "n") eq "y" ? true : false,
!$bChecksum,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_MAX),
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_ARCHIVE_REQUIRED, true, "y") eq "y" ? true : false,
config_load(CONFIG_SECTION_BACKUP, CONFIG_KEY_THREAD_TIMEOUT)
);
####################################################################################################################################
# BACKUP
####################################################################################################################################
if ($strOperation eq OP_BACKUP)
{
my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-backup.lock");
if (!lock_file_create($strLockFile))
{
&log(DEBUG, "backup process is already running for stanza ${strStanza} - exiting");
exit 0
}
backup(config_load(CONFIG_SECTION_STANZA, CONFIG_KEY_PATH));
$strOperation = OP_EXPIRE;
sleep(30);
lock_file_remove();
}
####################################################################################################################################
# EXPIRE
####################################################################################################################################
if ($strOperation eq OP_EXPIRE)
{
my $strLockFile = $oFile->path_get(PATH_BACKUP, "lock/${strStanza}-expire.lock");
if (!lock_file_create($strLockFile))
{
&log(DEBUG, "expire process is already running for stanza ${strStanza} - exiting");
exit 0
}
backup_expire
(
$oFile->path_get(PATH_BACKUP_CLUSTER),
config_load(CONFIG_SECTION_RETENTION, "full_retention"),
config_load(CONFIG_SECTION_RETENTION, "differential_retention"),
config_load(CONFIG_SECTION_RETENTION, "archive_retention_type"),
config_load(CONFIG_SECTION_RETENTION, "archive_retention")
);
lock_file_remove();
exit 0;
}
confess &log(ASSERT, "invalid operation ${strOperation} - missing handler block");

1591
pg_backrest_backup.pm Normal file
View File

File diff suppressed because it is too large Load Diff

146
pg_backrest_db.pm Normal file
View File

@ -0,0 +1,146 @@
####################################################################################################################################
# DB MODULE
####################################################################################################################################
package pg_backrest_db;
use threads;
use Moose;
use strict;
use warnings;
use Carp;
use Net::OpenSSH;
use File::Basename;
use IPC::System::Simple qw(capture);
use lib dirname($0);
use pg_backrest_utility;
# Command strings
has strCommandPsql => (is => 'bare'); # PSQL command
# Module variables
has strDbUser => (is => 'ro'); # Database user
has strDbHost => (is => 'ro'); # Database host
has oDbSSH => (is => 'bare'); # Database SSH object
has fVersion => (is => 'ro'); # Database version
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub BUILD
{
my $self = shift;
# Connect SSH object if db host is defined
if (defined($self->{strDbHost}) && !defined($self->{oDbSSH}))
{
&log(TRACE, "connecting to database ssh host $self->{strDbHost}");
# !!! This could be improved by redirecting stderr to a file to get a better error message
$self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, master_stderr_discard => true, user => $self->{strDbUser});
$self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
}
}
####################################################################################################################################
# IS_REMOTE
#
# Determine whether database operations are remote.
####################################################################################################################################
sub is_remote
{
my $self = shift;
# If the SSH object is defined then db is remote
return defined($self->{oDbSSH}) ? true : false;
}
####################################################################################################################################
# PSQL_EXECUTE
####################################################################################################################################
sub psql_execute
{
my $self = shift;
my $strScript = shift; # psql script to execute
# Get the user-defined command for psql
my $strCommand = $self->{strCommandPsql} . " -c \"${strScript}\" postgres";
my $strResult;
# !!! Need to capture error output with open3 and log it
# Run remotely
if ($self->is_remote())
{
&log(TRACE, "psql execute: remote ${strScript}");
$strResult = $self->{oDbSSH}->capture($strCommand)
or confess &log(ERROR, "unable to execute remote psql command '${strCommand}'");
}
# Else run locally
else
{
&log(TRACE, "psql execute: ${strScript}");
$strResult = capture($strCommand) or confess &log(ERROR, "unable to execute local psql command '${strCommand}'");
}
return $strResult;
}
####################################################################################################################################
# TABLESPACE_MAP_GET - Get the mapping between oid and tablespace name
####################################################################################################################################
sub tablespace_map_get
{
my $self = shift;
return data_hash_build("oid\tname\n" . $self->psql_execute(
"copy (select oid, spcname from pg_tablespace) to stdout"), "\t");
}
####################################################################################################################################
# VERSION_GET
####################################################################################################################################
sub version_get
{
my $self = shift;
if (defined($self->{fVersion}))
{
return $self->{fVersion};
}
$self->{fVersion} =
trim($self->psql_execute("copy (select (regexp_matches(split_part(version(), ' ', 2), '^[0-9]+\.[0-9]+'))[1]) to stdout"));
&log(DEBUG, "database version is $self->{fVersion}");
return $self->{fVersion};
}
####################################################################################################################################
# BACKUP_START
####################################################################################################################################
sub backup_start
{
my $self = shift;
my $strLabel = shift;
return trim($self->psql_execute("set client_min_messages = 'warning';" .
"copy (select pg_xlogfile_name(xlog) from pg_start_backup('${strLabel}') as xlog) to stdout"));
}
####################################################################################################################################
# BACKUP_STOP
####################################################################################################################################
sub backup_stop
{
my $self = shift;
return trim($self->psql_execute("set client_min_messages = 'warning';" .
"copy (select pg_xlogfile_name(xlog) from pg_stop_backup() as xlog) to stdout"))
}
no Moose;
__PACKAGE__->meta->make_immutable;

921
pg_backrest_file.pm Normal file
View File

@ -0,0 +1,921 @@
####################################################################################################################################
# FILE MODULE
####################################################################################################################################
package pg_backrest_file;
use threads;
use Moose;
use strict;
use warnings;
use Carp;
use Net::OpenSSH;
use IPC::Open3;
use File::Basename;
use IPC::System::Simple qw(capture);
use lib dirname($0);
use pg_backrest_utility;
use Exporter qw(import);
our @EXPORT = qw(PATH_DB PATH_DB_ABSOLUTE PATH_BACKUP PATH_BACKUP_ABSOLUTE PATH_BACKUP_CLUSTER PATH_BACKUP_TMP PATH_BACKUP_ARCHIVE);
# Extension and permissions
has strCompressExtension => (is => 'ro', default => 'gz');
has strDefaultPathPermission => (is => 'bare', default => '0750');
has strDefaultFilePermission => (is => 'ro', default => '0640');
# Command strings
has strCommandChecksum => (is => 'bare');
has strCommandCompress => (is => 'bare');
has strCommandDecompress => (is => 'bare');
has strCommandCat => (is => 'bare', default => 'cat %file%');
has strCommandManifest => (is => 'bare');
# Module variables
has strDbUser => (is => 'bare'); # Database user
has strDbHost => (is => 'bare'); # Database host
has oDbSSH => (is => 'bare'); # Database SSH object
has strBackupUser => (is => 'bare'); # Backup user
has strBackupHost => (is => 'bare'); # Backup host
has oBackupSSH => (is => 'bare'); # Backup SSH object
has strBackupPath => (is => 'bare'); # Backup base path
has strBackupClusterPath => (is => 'bare'); # Backup cluster path
# Process flags
has bNoCompression => (is => 'bare');
has strStanza => (is => 'bare');
has iThreadIdx => (is => 'bare');
####################################################################################################################################
# CONSTRUCTOR
####################################################################################################################################
sub BUILD
{
my $self = shift;
# Make sure the backup path is defined
if (!defined($self->{strBackupPath}))
{
confess &log(ERROR, "common:backup_path undefined");
}
# Create the backup cluster path
$self->{strBackupClusterPath} = $self->{strBackupPath} . "/" . $self->{strStanza};
# Create the ssh options string
if (defined($self->{strBackupHost}) || defined($self->{strDbHost}))
{
my $strOptionSSH = "Compression=no";
if ($self->{bNoCompression})
{
$strOptionSSH = "Compression=yes";
}
# Connect SSH object if backup host is defined
if (!defined($self->{oBackupSSH}) && defined($self->{strBackupHost}))
{
&log(TRACE, "connecting to backup ssh host " . $self->{strBackupHost});
# !!! This could be improved by redirecting stderr to a file to get a better error message
$self->{oBackupSSH} = Net::OpenSSH->new($self->{strBackupHost}, timeout => 300, user => $self->{strBackupUser}, master_opts => [-o => $strOptionSSH]);
$self->{oBackupSSH}->error and confess &log(ERROR, "unable to connect to $self->{strBackupHost}: " . $self->{oBackupSSH}->error);
}
# Connect SSH object if db host is defined
if (!defined($self->{oDbSSH}) && defined($self->{strDbHost}))
{
&log(TRACE, "connecting to database ssh host $self->{strDbHost}");
# !!! This could be improved by redirecting stderr to a file to get a better error message
$self->{oDbSSH} = Net::OpenSSH->new($self->{strDbHost}, timeout => 300, user => $self->{strDbUser}, master_opts => [-o => $strOptionSSH]);
$self->{oDbSSH}->error and confess &log(ERROR, "unable to connect to $self->{strDbHost}: " . $self->{oDbSSH}->error);
}
}
}
####################################################################################################################################
# CLONE
####################################################################################################################################
sub clone
{
my $self = shift;
my $iThreadIdx = shift;
return pg_backrest_file->new
(
strCompressExtension => $self->{strCompressExtension},
strDefaultPathPermission => $self->{strDefaultPathPermission},
strDefaultFilePermission => $self->{strDefaultFilePermission},
strCommandChecksum => $self->{strCommandChecksum},
strCommandCompress => $self->{strCommandCompress},
strCommandDecompress => $self->{strCommandDecompress},
strCommandCat => $self->{strCommandCat},
strCommandManifest => $self->{strCommandManifest},
# oDbSSH => $self->{strDbSSH},
strDbUser => $self->{strDbUser},
strDbHost => $self->{strDbHost},
# oBackupSSH => $self->{strBackupSSH},
strBackupUser => $self->{strBackupUser},
strBackupHost => $self->{strBackupHost},
strBackupPath => $self->{strBackupPath},
strBackupClusterPath => $self->{strBackupClusterPath},
bNoCompression => $self->{bNoCompression},
strStanza => $self->{strStanza},
iThreadIdx => $iThreadIdx
);
}
####################################################################################################################################
# PATH_GET
####################################################################################################################################
use constant
{
PATH_DB => 'db',
PATH_DB_ABSOLUTE => 'db:absolute',
PATH_BACKUP => 'backup',
PATH_BACKUP_ABSOLUTE => 'backup:absolute',
PATH_BACKUP_CLUSTER => 'backup:cluster',
PATH_BACKUP_TMP => 'backup:tmp',
PATH_BACKUP_ARCHIVE => 'backup:archive'
};
sub path_type_get
{
my $self = shift;
my $strType = shift;
# If db type
if ($strType =~ /^db(\:.*){0,1}/)
{
return PATH_DB;
}
# Else if backup type
elsif ($strType =~ /^backup(\:.*){0,1}/)
{
return PATH_BACKUP;
}
# Error when path type not recognized
confess &log(ASSERT, "no known path types in '${strType}'");
}
sub path_get
{
my $self = shift;
my $strType = shift; # Base type of the path to get (PATH_DB_ABSOLUTE, PATH_BACKUP_TMP, etc)
my $strFile = shift; # File to append to the base path (can include a path as well)
my $bTemp = shift; # Return the temp file for this path type - only some types have temp files
# Only allow temp files for PATH_BACKUP_ARCHIVE and PATH_BACKUP_TMP
if (defined($bTemp) && $bTemp && !($strType eq PATH_BACKUP_ARCHIVE || $strType eq PATH_BACKUP_TMP))
{
confess &log(ASSERT, "temp file not supported on path " . $strType);
}
# Get absolute db path
if ($strType eq PATH_DB_ABSOLUTE)
{
return $strFile;
}
# Make sure the base backup path is defined
if (!defined($self->{strBackupPath}))
{
confess &log(ASSERT, "\$strBackupPath not yet defined");
}
# Get absolute backup path
if ($strType eq PATH_BACKUP_ABSOLUTE)
{
# Need a check in here to make sure this is relative to the backup path
return $strFile;
}
# Get base backup path
if ($strType eq PATH_BACKUP)
{
return $self->{strBackupPath} . (defined($strFile) ? "/${strFile}" : "");
}
# Make sure the cluster is defined
if (!defined($self->{strStanza}))
{
confess &log(ASSERT, "\$strStanza not yet defined");
}
# Get the backup tmp path
if ($strType eq PATH_BACKUP_TMP)
{
my $strTempPath = "$self->{strBackupPath}/temp/$self->{strStanza}.tmp";
if (defined($bTemp) && $bTemp)
{
return "${strTempPath}/file.tmp" . (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : "");
}
return "${strTempPath}" . (defined($strFile) ? "/${strFile}" : "");
}
# Get the backup archive path
if ($strType eq PATH_BACKUP_ARCHIVE)
{
my $strArchivePath = "$self->{strBackupPath}/archive/$self->{strStanza}";
my $strArchive;
if (defined($bTemp) && $bTemp)
{
return "${strArchivePath}/file.tmp" . (defined($self->{iThreadIdx}) ? ".$self->{iThreadIdx}" : "");
}
if (defined($strFile))
{
$strArchive = substr(basename($strFile), 0, 24);
if ($strArchive !~ /^([0-F]){24}$/)
{
return "${strArchivePath}/${strFile}";
}
}
return $strArchivePath . (defined($strArchive) ? "/" . substr($strArchive, 0, 16) : "") .
(defined($strFile) ? "/" . $strFile : "");
}
if ($strType eq PATH_BACKUP_CLUSTER)
{
return $self->{strBackupPath} . "/backup/$self->{strStanza}" . (defined($strFile) ? "/${strFile}" : "");
}
# Error when path type not recognized
confess &log(ASSERT, "no known path types in '${strType}'");
}
####################################################################################################################################
# LINK_CREATE
####################################################################################################################################
sub link_create
{
my $self = shift;
my $strSourcePathType = shift;
my $strSourceFile = shift;
my $strDestinationPathType = shift;
my $strDestinationFile = shift;
my $bHard = shift;
my $bRelative = shift;
my $bPathCreate = shift;
# if bHard is not defined default to false
$bHard = defined($bHard) ? $bHard : false;
# if bRelative is not defined or bHard is true, default to false
$bRelative = !defined($bRelative) || $bHard ? false : $bRelative;
# if bPathCreate is not defined, default to true
$bPathCreate = defined($bPathCreate) ? $bPathCreate : true;
# Source and destination path types must be the same (both PATH_DB or both PATH_BACKUP)
if ($self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType))
{
confess &log(ASSERT, "path types must be equal in link create");
}
# Generate source and destination files
my $strSource = $self->path_get($strSourcePathType, $strSourceFile);
my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile);
# If the destination path is backup and does not exist, create it
if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP)
{
$self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination));
}
unless (-e $strSource)
{
if (-e $strSource . ".$self->{strCompressExtension}")
{
$strSource .= ".$self->{strCompressExtension}";
$strDestination .= ".$self->{strCompressExtension}";
}
else
{
# Error when a hardlink will be created on a missing file
if ($bHard)
{
confess &log(ASSERT, "unable to find ${strSource}(.$self->{strCompressExtension}) for link");
}
}
}
# Generate relative path if requested
if ($bRelative)
{
my $iCommonLen = common_prefix($strSource, $strDestination);
if ($iCommonLen != 0)
{
$strSource = ("../" x substr($strDestination, $iCommonLen) =~ tr/\///) . substr($strSource, $iCommonLen);
}
}
# Create the command
my $strCommand = "ln" . (!$bHard ? " -s" : "") . " ${strSource} ${strDestination}";
# Run remotely
if ($self->is_remote($strSourcePathType))
{
&log(TRACE, "link_create: remote ${strSourcePathType} '${strCommand}'");
my $oSSH = $self->remote_get($strSourcePathType);
$oSSH->system($strCommand) or confess &log("unable to create link from ${strSource} to ${strDestination}");
}
# Run locally
else
{
&log(TRACE, "link_create: local '${strCommand}'");
system($strCommand) == 0 or confess &log("unable to create link from ${strSource} to ${strDestination}");
}
}
####################################################################################################################################
# PATH_CREATE
#
# Creates a path locally or remotely. Currently does not error if the path already exists. Also does not set permissions if the
# path aleady exists.
####################################################################################################################################
sub path_create
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
my $strPermission = shift;
# If no permissions are given then use the default
if (!defined($strPermission))
{
$strPermission = $self->{strDefaultPathPermission};
}
# Get the path to create
my $strPathCreate = $strPath;
if (defined($strPathType))
{
$strPathCreate = $self->path_get($strPathType, $strPath);
}
my $strCommand = "mkdir -p -m ${strPermission} ${strPathCreate}";
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "path_create: remote ${strPathType} '${strCommand}'");
my $oSSH = $self->remote_get($strPathType);
$oSSH->system($strCommand) or confess &log("unable to create remote path ${strPathType}:${strPath}");
}
# Run locally
else
{
&log(TRACE, "path_create: local '${strCommand}'");
system($strCommand) == 0 or confess &log(ERROR, "unable to create path ${strPath}");
}
}
####################################################################################################################################
# IS_REMOTE
#
# Determine whether any operations are being performed remotely. If $strPathType is defined, the function will return true if that
# path is remote. If $strPathType is not defined, then function will return true if any path is remote.
####################################################################################################################################
sub is_remote
{
my $self = shift;
my $strPathType = shift;
# If the SSH object is defined then some paths are remote
if (defined($self->{oDbSSH}) || defined($self->{oBackupSSH}))
{
# If path type is not defined but the SSH object is, then some paths are remote
if (!defined($strPathType))
{
return true;
}
# If a host is defined for the path then it is remote
if (defined($self->{strBackupHost}) && $self->path_type_get($strPathType) eq PATH_BACKUP ||
defined($self->{strDbHost}) && $self->path_type_get($strPathType) eq PATH_DB)
{
return true;
}
}
return false;
}
####################################################################################################################################
# REMOTE_GET
#
# Get remote SSH object depending on the path type.
####################################################################################################################################
sub remote_get
{
my $self = shift;
my $strPathType = shift;
# Get the db SSH object
if ($self->path_type_get($strPathType) eq PATH_DB && defined($self->{oDbSSH}))
{
return $self->{oDbSSH};
}
# Get the backup SSH object
if ($self->path_type_get($strPathType) eq PATH_BACKUP && defined($self->{oBackupSSH}))
{
return $self->{oBackupSSH}
}
# Error when no ssh object is found
confess &log(ASSERT, "path type ${strPathType} does not have a defined ssh object");
}
####################################################################################################################################
# FILE_MOVE
#
# Moves a file locally or remotely.
####################################################################################################################################
sub file_move
{
my $self = shift;
my $strSourcePathType = shift;
my $strSourceFile = shift;
my $strDestinationPathType = shift;
my $strDestinationFile = shift;
my $bPathCreate = shift;
# if bPathCreate is not defined, default to true
$bPathCreate = defined($bPathCreate) ? $bPathCreate : true;
&log(TRACE, "file_move: ${strSourcePathType}: " . (defined($strSourceFile) ? ":${strSourceFile}" : "") .
" to ${strDestinationPathType}" . (defined($strDestinationFile) ? ":${strDestinationFile}" : ""));
# Get source and desination files
if ($self->path_type_get($strSourcePathType) ne $self->path_type_get($strSourcePathType))
{
confess &log(ASSERT, "source and destination path types must be equal");
}
my $strSource = $self->path_get($strSourcePathType, $strSourceFile);
my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile);
# If the destination path is backup and does not exist, create it
if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP)
{
$self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination));
}
my $strCommand = "mv ${strSource} ${strDestination}";
# Run remotely
if ($self->is_remote($strDestinationPathType))
{
&log(TRACE, "file_move: remote ${strDestinationPathType} '${strCommand}'");
my $oSSH = $self->remote_get($strDestinationPathType);
$oSSH->system($strCommand)
or confess &log("unable to move remote ${strDestinationPathType}:${strSourceFile} to ${strDestinationFile}");
}
# Run locally
else
{
&log(TRACE, "file_move: '${strCommand}'");
system($strCommand) == 0 or confess &log("unable to move local ${strSourceFile} to ${strDestinationFile}");
}
}
####################################################################################################################################
# FILE_COPY
####################################################################################################################################
sub file_copy
{
my $self = shift;
my $strSourcePathType = shift;
my $strSourceFile = shift;
my $strDestinationPathType = shift;
my $strDestinationFile = shift;
my $bNoCompressionOverride = shift;
my $lModificationTime = shift;
my $strPermission = shift;
my $bPathCreate = shift;
my $bConfessCopyError = shift;
# if bPathCreate is not defined, default to true
$bPathCreate = defined($bPathCreate) ? $bPathCreate : true;
$bConfessCopyError = defined($bConfessCopyError) ? $bConfessCopyError : false;
&log(TRACE, "file_copy: ${strSourcePathType}: " . (defined($strSourceFile) ? ":${strSourceFile}" : "") .
" to ${strDestinationPathType}" . (defined($strDestinationFile) ? ":${strDestinationFile}" : ""));
# Modification time and permissions cannot be set remotely
if ((defined($lModificationTime) || defined($strPermission)) && $self->is_remote($strDestinationPathType))
{
confess &log(ASSERT, "modification time and permissions cannot be set on remote destination file");
}
# Generate source, destination and tmp filenames
my $strSource = $self->path_get($strSourcePathType, $strSourceFile);
my $strDestination = $self->path_get($strDestinationPathType, $strDestinationFile);
my $strDestinationTmp = $self->path_get($strDestinationPathType, $strDestinationFile, true);
# Is this already a compressed file?
my $bAlreadyCompressed = $strSource =~ "^.*\.$self->{strCompressExtension}\$";
if ($bAlreadyCompressed && $strDestination !~ "^.*\.$self->{strCompressExtension}\$")
{
$strDestination .= ".$self->{strCompressExtension}";
}
# Does the file need compression?
my $bCompress = !((defined($bNoCompressionOverride) && $bNoCompressionOverride) ||
(!defined($bNoCompressionOverride) && $self->{bNoCompression}));
# If the destination path is backup and does not exist, create it
if ($bPathCreate && $self->path_type_get($strDestinationPathType) eq PATH_BACKUP)
{
$self->path_create(PATH_BACKUP_ABSOLUTE, dirname($strDestination));
}
# Generate the command string depending on compression/decompression/cat
my $strCommand = $self->{strCommandCat};
if (!$bAlreadyCompressed && $bCompress)
{
$strCommand = $self->{strCommandCompress};
$strDestination .= ".gz";
}
elsif ($bAlreadyCompressed && !$bCompress)
{
$strCommand = $self->{strCommandDecompress};
$strDestination = substr($strDestination, 0, length($strDestination) - length($self->{strCompressExtension}) - 1);
}
$strCommand =~ s/\%file\%/${strSource}/g;
$strCommand .= " 2> /dev/null";
# If this command is remote on only one side
if ($self->is_remote($strSourcePathType) && !$self->is_remote($strDestinationPathType) ||
!$self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType))
{
# Else if the source is remote
if ($self->is_remote($strSourcePathType))
{
&log(TRACE, "file_copy: remote ${strSource} to local ${strDestination}");
# Open the destination file for writing (will be streamed from the ssh session)
my $hFile;
open($hFile, ">", $strDestinationTmp) or confess &log(ERROR, "cannot open ${strDestination}");
# Execute the command through ssh
my $oSSH = $self->remote_get($strSourcePathType);
unless ($oSSH->system({stdout_fh => $hFile}, $strCommand))
{
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
my $strResult = "unable to execute ssh '${strCommand}'";
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
# Close the destination file handle
close($hFile) or confess &log(ERROR, "cannot close file ${strDestinationTmp}");
}
# Else if the destination is remote
elsif ($self->is_remote($strDestinationPathType))
{
&log(TRACE, "file_copy: local ${strSource} ($strCommand) to remote ${strDestination}");
# Open the input command as a stream
my $hOut;
my $pId = open3(undef, $hOut, undef, $strCommand) or confess(ERROR, "unable to execute '${strCommand}'");
# Execute the command though ssh
my $oSSH = $self->remote_get($strDestinationPathType);
$oSSH->system({stdin_fh => $hOut}, "cat > ${strDestinationTmp}") or confess &log(ERROR, "unable to execute ssh 'cat'");
# Wait for the stream process to finish
waitpid($pId, 0);
my $iExitStatus = ${^CHILD_ERROR_NATIVE} >> 8;
if ($iExitStatus != 0)
{
my $strResult = "command '${strCommand}' returned " . $iExitStatus;
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
}
# If the source and destination are both remote but not the same remote
elsif ($self->is_remote($strSourcePathType) && $self->is_remote($strDestinationPathType) &&
$self->path_type_get($strSourcePathType) ne $self->path_type_get($strDestinationPathType))
{
&log(TRACE, "file_copy: remote ${strSource} to remote ${strDestination}");
confess &log(ASSERT, "remote source and destination not supported");
}
# Else this is a local command or remote where both sides are the same remote
else
{
# Complete the command by redirecting to the destination tmp file
$strCommand .= " > ${strDestinationTmp}";
if ($self->is_remote($strSourcePathType))
{
&log(TRACE, "file_copy: remote ${strSourcePathType} '${strCommand}'");
my $oSSH = $self->remote_get($strSourcePathType);
unless($oSSH->system($strCommand))
{
my $strResult = "unable to execute remote command ${strCommand}:" . oSSH->error;
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
else
{
&log(TRACE, "file_copy: local '${strCommand}'");
unless(system($strCommand) == 0)
{
my $strResult = "unable to copy local ${strSource} to local ${strDestinationTmp}";
$bConfessCopyError ? confess &log(ERROR, $strResult) : return false;
}
}
}
# Set the file permission if required (this only works locally for now)
if (defined($strPermission))
{
&log(TRACE, "file_copy: chmod ${strPermission}");
system("chmod ${strPermission} ${strDestinationTmp}") == 0
or confess &log(ERROR, "unable to set permissions for local ${strDestinationTmp}");
}
# Set the file modification time if required (this only works locally for now)
if (defined($lModificationTime))
{
&log(TRACE, "file_copy: time ${lModificationTime}");
utime($lModificationTime, $lModificationTime, $strDestinationTmp)
or confess &log(ERROR, "unable to set time for local ${strDestinationTmp}");
}
# Move the file from tmp to final destination
$self->file_move($self->path_type_get($strSourcePathType) . ":absolute", $strDestinationTmp,
$self->path_type_get($strDestinationPathType) . ":absolute", $strDestination, $bPathCreate);
return true;
}
####################################################################################################################################
# FILE_HASH_GET
####################################################################################################################################
sub file_hash_get
{
my $self = shift;
my $strPathType = shift;
my $strFile = shift;
# For now this operation is not supported remotely. Not currently needed.
if ($self->is_remote($strPathType))
{
confess &log(ASSERT, "remote operation not supported");
}
if (!defined($self->{strCommandChecksum}))
{
confess &log(ASSERT, "\$strCommandChecksum not defined");
}
my $strPath = $self->path_get($strPathType, $strFile);
my $strCommand;
if (-e $strPath)
{
$strCommand = $self->{strCommandChecksum};
$strCommand =~ s/\%file\%/${strPath}/g;
}
elsif (-e $strPath . ".$self->{strCompressExtension}")
{
$strCommand = $self->{strCommandDecompress};
$strCommand =~ s/\%file\%/${strPath}/g;
$strCommand .= " | " . $self->{strCommandChecksum};
$strCommand =~ s/\%file\%//g;
}
else
{
confess &log(ASSERT, "unable to find $strPath(.$self->{strCompressExtension}) for checksum");
}
return trim(capture($strCommand)) or confess &log(ERROR, "unable to checksum ${strPath}");
}
####################################################################################################################################
# FILE_COMPRESS
####################################################################################################################################
sub file_compress
{
my $self = shift;
my $strPathType = shift;
my $strFile = shift;
# For now this operation is not supported remotely. Not currently needed.
if ($self->is_remote($strPathType))
{
confess &log(ASSERT, "remote operation not supported");
}
if (!defined($self->{strCommandCompress}))
{
confess &log(ASSERT, "\$strCommandChecksum not defined");
}
my $strPath = $self->path_get($strPathType, $strFile);
# Build the command
my $strCommand = $self->{strCommandCompress};
$strCommand =~ s/\%file\%/${strPath}/g;
$strCommand =~ s/\ \-\-stdout//g;
system($strCommand) == 0 or confess &log(ERROR, "unable to compress ${strPath}: ${strCommand}");
}
####################################################################################################################################
# FILE_LIST_GET
####################################################################################################################################
sub file_list_get
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
my $strExpression = shift;
my $strSortOrder = shift;
# For now this operation is not supported remotely. Not currently needed.
if ($self->is_remote($strPathType))
{
confess &log(ASSERT, "remote operation not supported");
}
my $strPathList = $self->path_get($strPathType, $strPath);
my $hDir;
opendir $hDir, $strPathList or confess &log(ERROR, "unable to open path ${strPathList}");
my @stryFileAll = readdir $hDir or confess &log(ERROR, "unable to get files for path ${strPathList}, expression ${strExpression}");
close $hDir;
my @stryFile;
if (@stryFileAll)
{
@stryFile = grep(/$strExpression/i, @stryFileAll)
}
if (@stryFile)
{
if (defined($strSortOrder) && $strSortOrder eq "reverse")
{
return sort {$b cmp $a} @stryFile;
}
else
{
return sort @stryFile;
}
}
return @stryFile;
}
####################################################################################################################################
# FILE_EXISTS
####################################################################################################################################
sub file_exists
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
# Get the root path for the manifest
my $strPathExists = $self->path_get($strPathType, $strPath);
# Builds the exists command
my $strCommand = "ls ${strPathExists} 2> /dev/null";
# Run the file exists command
my $strExists = "";
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "file_exists: remote ${strPathType}:${strPathExists}");
my $oSSH = $self->remote_get($strPathType);
$strExists = $oSSH->capture($strCommand);
}
# Run locally
else
{
&log(TRACE, "file_exists: local ${strPathType}:${strPathExists}");
$strExists = capture($strCommand);
}
# If the return from ls eq strPathExists then true
return ($strExists eq $strPathExists);
}
####################################################################################################################################
# FILE_REMOVE
####################################################################################################################################
sub file_remove
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
my $bTemp = shift;
my $bErrorIfNotExists = shift;
if (!defined($bErrorIfNotExists))
{
$bErrorIfNotExists = false;
}
# Get the root path for the manifest
my $strPathRemove = $self->path_get($strPathType, $strPath, $bTemp);
# Builds the exists command
my $strCommand = "rm -f ${strPathRemove}";
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "file_remove: remote ${strPathType}:${strPathRemove}");
my $oSSH = $self->remote_get($strPathType);
$oSSH->system($strCommand) or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove remote ${strPathType}:${strPathRemove}") : true;
}
# Run locally
else
{
&log(TRACE, "file_exists: local ${strPathType}:${strPathRemove}");
system($strCommand) == 0 or $bErrorIfNotExists ? confess &log(ERROR, "unable to remove local ${strPathType}:${strPathRemove}") : true;
}
}
####################################################################################################################################
# MANIFEST_GET
#
# Builds a path/file manifest starting with the base path and including all subpaths. The manifest contains all the information
# needed to perform a backup or a delta with a previous backup.
####################################################################################################################################
sub manifest_get
{
my $self = shift;
my $strPathType = shift;
my $strPath = shift;
&log(TRACE, "manifest: " . $self->{strCommandManifest});
# Get the root path for the manifest
my $strPathManifest = $self->path_get($strPathType, $strPath);
# Builds the manifest command
my $strCommand = $self->{strCommandManifest};
$strCommand =~ s/\%path\%/${strPathManifest}/g;
$strCommand .= " 2> /dev/null";
# Run the manifest command
my $strManifest;
# Run remotely
if ($self->is_remote($strPathType))
{
&log(TRACE, "manifest_get: remote ${strPathType}:${strPathManifest}");
my $oSSH = $self->remote_get($strPathType);
$strManifest = $oSSH->capture($strCommand) or confess &log(ERROR, "unable to execute remote command '${strCommand}'");
}
# Run locally
else
{
&log(TRACE, "manifest_get: local ${strPathType}:${strPathManifest}");
$strManifest = capture($strCommand) or confess &log(ERROR, "unable to execute local command '${strCommand}'");
}
# Load the manifest into a hash
return data_hash_build("name\ttype\tuser\tgroup\tpermission\tmodification_time\tinode\tsize\tlink_destination\n" .
$strManifest, "\t", ".");
}
no Moose;
__PACKAGE__->meta->make_immutable;

354
pg_backrest_utility.pm Normal file
View File

@ -0,0 +1,354 @@
####################################################################################################################################
# UTILITY MODULE
####################################################################################################################################
package pg_backrest_utility;
use threads;
use strict;
use warnings;
use Carp;
use IPC::System::Simple qw(capture);
use Fcntl qw(:DEFAULT :flock);
use Exporter qw(import);
our @EXPORT = qw(data_hash_build trim common_prefix wait_for_file date_string_get file_size_format execute
log log_file_set log_level_set
lock_file_create lock_file_remove
TRACE DEBUG ERROR ASSERT WARN INFO true false);
# Global constants
use constant
{
true => 1,
false => 0
};
use constant
{
TRACE => 'TRACE',
DEBUG => 'DEBUG',
INFO => 'INFO',
WARN => 'WARN',
ERROR => 'ERROR',
ASSERT => 'ASSERT',
OFF => 'OFF'
};
my $hLogFile;
my $strLogLevelFile = ERROR;
my $strLogLevelConsole = ERROR;
my %oLogLevelRank;
my $strLockFile;
my $hLockFile;
$oLogLevelRank{TRACE}{rank} = 6;
$oLogLevelRank{DEBUG}{rank} = 5;
$oLogLevelRank{INFO}{rank} = 4;
$oLogLevelRank{WARN}{rank} = 3;
$oLogLevelRank{ERROR}{rank} = 2;
$oLogLevelRank{ASSERT}{rank} = 1;
$oLogLevelRank{OFF}{rank} = 0;
####################################################################################################################################
# LOCK_FILE_CREATE
####################################################################################################################################
sub lock_file_create
{
my $strLockFileParam = shift;
$strLockFile = $strLockFileParam;
if (defined($hLockFile))
{
confess &lock(ASSERT, "${strLockFile} lock is already held, cannot create lock ${strLockFile}");
}
sysopen($hLockFile, $strLockFile, O_WRONLY | O_CREAT)
or confess &log(ERROR, "unable to open lock file ${strLockFile}");
if (!flock($hLockFile, LOCK_EX | LOCK_NB))
{
close($hLockFile);
return 0;
}
return $hLockFile;
}
####################################################################################################################################
# LOCK_FILE_REMOVE
####################################################################################################################################
sub lock_file_remove
{
if (defined($hLockFile))
{
close($hLockFile);
unlink($strLockFile) or confess &log(ERROR, "unable to remove lock file ${strLockFile}");
$hLockFile = undef;
$strLockFile = undef;
}
else
{
confess &log(ASSERT, "there is no lock to free");
}
}
####################################################################################################################################
# DATA_HASH_BUILD - Hash a delimited file with header
####################################################################################################################################
sub data_hash_build
{
my $strData = shift;
my $strDelimiter = shift;
my $strUndefinedKey = shift;
my @stryFile = split("\n", $strData);
my @stryHeader = split($strDelimiter, $stryFile[0]);
my %oHash;
for (my $iLineIdx = 1; $iLineIdx < scalar @stryFile; $iLineIdx++)
{
my @stryLine = split($strDelimiter, $stryFile[$iLineIdx]);
if (!defined($stryLine[0]) || $stryLine[0] eq "")
{
$stryLine[0] = $strUndefinedKey;
}
for (my $iColumnIdx = 1; $iColumnIdx < scalar @stryHeader; $iColumnIdx++)
{
if (defined($oHash{"$stryHeader[0]"}{"$stryLine[0]"}{"$stryHeader[$iColumnIdx]"}))
{
confess "the first column must be unique to build the hash";
}
$oHash{"$stryHeader[0]"}{"$stryLine[0]"}{"$stryHeader[$iColumnIdx]"} = $stryLine[$iColumnIdx];
}
}
return %oHash;
}
####################################################################################################################################
# TRIM - trim whitespace off strings
####################################################################################################################################
sub trim
{
my $strBuffer = shift;
$strBuffer =~ s/^\s+|\s+$//g;
return $strBuffer;
}
####################################################################################################################################
# WAIT_FOR_FILE
####################################################################################################################################
sub wait_for_file
{
my $strDir = shift;
my $strRegEx = shift;
my $iSeconds = shift;
my $lTime = time();
my $hDir;
while ($lTime > time() - $iSeconds)
{
opendir $hDir, $strDir or die "Could not open dir: $!\n";
my @stryFile = grep(/$strRegEx/i, readdir $hDir);
close $hDir;
if (scalar @stryFile == 1)
{
return;
}
sleep(1);
}
confess &log(ERROR, "could not find $strDir/$strRegEx after $iSeconds second(s)");
}
####################################################################################################################################
# COMMON_PREFIX
####################################################################################################################################
sub common_prefix
{
my $strString1 = shift;
my $strString2 = shift;
my $iCommonLen = 0;
my $iCompareLen = length($strString1) < length($strString2) ? length($strString1) : length($strString2);
for (my $iIndex = 0; $iIndex < $iCompareLen; $iIndex++)
{
if (substr($strString1, $iIndex, 1) ne substr($strString2, $iIndex, 1))
{
last;
}
$iCommonLen ++;
}
return $iCommonLen;
}
####################################################################################################################################
# FILE_SIZE_FORMAT - Format file sizes in human-readable form
####################################################################################################################################
sub file_size_format
{
my $lFileSize = shift;
if ($lFileSize < 1024)
{
return $lFileSize . "B";
}
if ($lFileSize < (1024 * 1024))
{
return int($lFileSize / 1024) . "KB";
}
if ($lFileSize < (1024 * 1024 * 1024))
{
return int($lFileSize / 1024 / 1024) . "MB";
}
return int($lFileSize / 1024 / 1024 / 1024) . "GB";
}
####################################################################################################################################
# DATE_STRING_GET - Get the date and time string
####################################################################################################################################
sub date_string_get
{
my $strFormat = shift;
if (!defined($strFormat))
{
$strFormat = "%4d%02d%02d-%02d%02d%02d";
}
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
return(sprintf($strFormat, $year+1900, $mon+1, $mday, $hour, $min, $sec));
}
####################################################################################################################################
# LOG_FILE_SET - set the file messages will be logged to
####################################################################################################################################
sub log_file_set
{
my $strFile = shift;
$strFile .= "-" . date_string_get("%4d%02d%02d") . ".log";
my $bExists = false;
if (-e $strFile)
{
$bExists = true;
}
open($hLogFile, '>>', $strFile) or confess "unable to open log file ${strFile}";
if ($bExists)
{
print $hLogFile "\n";
}
print $hLogFile "-------------------PROCESS START-------------------\n";
}
####################################################################################################################################
# LOG_LEVEL_SET - set the log level for file and console
####################################################################################################################################
sub log_level_set
{
my $strLevelFileParam = shift;
my $strLevelConsoleParam = shift;
if (!defined($oLogLevelRank{"${strLevelFileParam}"}{rank}))
{
confess &log(ERROR, "file log level ${strLevelFileParam} does not exist");
}
if (!defined($oLogLevelRank{"${strLevelConsoleParam}"}{rank}))
{
confess &log(ERROR, "console log level ${strLevelConsoleParam} does not exist");
}
$strLogLevelFile = $strLevelFileParam;
$strLogLevelConsole = $strLevelConsoleParam;
}
####################################################################################################################################
# LOG - log messages
####################################################################################################################################
sub log
{
my $strLevel = shift;
my $strMessage = shift;
if (!defined($oLogLevelRank{"${strLevel}"}{rank}))
{
confess &log(ASSERT, "log level ${strLevel} does not exist");
}
my ($sec,$min,$hour,$mday,$mon,$year,$wday,$yday,$isdst) = localtime(time);
if (!defined($strMessage))
{
$strMessage = "(undefined)";
}
if ($strLevel eq "TRACE")
{
$strMessage = " " . $strMessage;
}
elsif ($strLevel eq "DEBUG")
{
$strMessage = " " . $strMessage;
}
$strMessage = sprintf("%4d-%02d-%02d %02d:%02d:%02d", $year+1900, $mon+1, $mday, $hour, $min, $sec) .
(" " x (7 - length($strLevel))) . "${strLevel} " . (" " x (2 - length(threads->tid()))) .
threads->tid() . ": ${strMessage}\n";
if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelConsole}"}{rank})
{
print $strMessage;
}
if ($oLogLevelRank{"${strLevel}"}{rank} <= $oLogLevelRank{"${strLogLevelFile}"}{rank})
{
if (defined($hLogFile))
{
print $hLogFile $strMessage;
}
}
return $strMessage;
}
####################################################################################################################################
# EXECUTE - execute a command
####################################################################################################################################
sub execute
{
my $strCommand = shift;
my $strOutput;
# print("$strCommand");
$strOutput = capture($strCommand) or confess &log(ERROR, "unable to execute command ${strCommand}: " . $_);
return $strOutput;
}
1;

244
test/test.pl Executable file
View File

@ -0,0 +1,244 @@
#!/usr/bin/perl
# /Library/PostgreSQL/9.3/bin/pg_ctl start -o "-c port=7000" -D /Users/backrest/test/backup/db/20140205-103801F/base -l /Users/backrest/test/backup/db/20140205-103801F/base/postgresql.log -w -s
#use strict;
use DBI;
use IPC::System::Simple qw(capture);
use Config::IniFiles;
use File::Find;
sub trim
{
local($strBuffer) = @_;
$strBuffer =~ s/^\s+|\s+$//g;
return $strBuffer;
}
sub execute
{
local($strCommand) = @_;
my $strOutput;
print("$strCommand");
$strOutput = trim(capture($strCommand));
if ($strOutput eq "")
{
print(" ... complete\n\n");
}
else
{
print(" ... complete\n$strOutput\n\n");
}
return $strOutput;
}
sub pg_create
{
local($strPgBinPath, $strTestPath, $strTestDir, $strArchiveDir, $strBackupDir) = @_;
execute("mkdir $strTestPath");
execute("mkdir $strTestPath/$strTestDir");
execute("mkdir $strTestPath/$strTestDir/ts1");
execute("mkdir $strTestPath/$strTestDir/ts2");
execute($strPgBinPath . "/initdb -D $strTestPath/$strTestDir/common -A trust -k");
execute("mkdir $strTestPath/$strBackupDir");
# execute("mkdir -p $strTestPath/$strArchiveDir");
}
sub pg_start
{
local($strPgBinPath, $strDbPath, $strPort, $strAchiveCommand) = @_;
my $strCommand = "$strPgBinPath/pg_ctl start -o \"-c port=$strPort -c checkpoint_segments=1 -c wal_level=archive -c archive_mode=on -c archive_command=\'$strAchiveCommand\'\" -D $strDbPath -l $strDbPath/postgresql.log -w -s";
execute($strCommand);
}
sub pg_password_set
{
local($strPgBinPath, $strPath, $strUser, $strPort) = @_;
my $strCommand = "$strPgBinPath/psql --port=$strPort -c \"alter user $strUser with password 'password'\" postgres";
execute($strCommand);
}
sub pg_stop
{
local($strPgBinPath, $strPath) = @_;
my $strCommand = "$strPgBinPath/pg_ctl stop -D $strPath -w -s -m fast";
execute($strCommand);
}
sub pg_drop
{
local($strTestPath) = @_;
my $strCommand = "rm -rf $strTestPath";
execute($strCommand);
}
sub pg_execute
{
local($dbh, $strSql) = @_;
print($strSql);
$sth = $dbh->prepare($strSql);
$sth->execute() or die;
$sth->finish();
print(" ... complete\n\n");
}
sub archive_command_build
{
my $strBackRestBinPath = shift;
my $strDestinationPath = shift;
my $bCompression = shift;
my $bChecksum = shift;
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --stanza=db --config=$strBackRestBinPath/pg_backrest.conf";
# if (!$bCompression)
# {
# $strCommand .= " --no-compression"
# }
#
# if (!$bChecksum)
# {
# $strCommand .= " --no-checksum"
# }
return $strCommand . " archive-push %p";
}
sub wait_for_file
{
my $strDir = shift;
my $strRegEx = shift;
my $iSeconds = shift;
my $lTime = time();
my $hDir;
while ($lTime > time() - $iSeconds)
{
opendir $hDir, $strDir or die "Could not open dir: $!\n";
my @stryFile = grep(/$strRegEx/i, readdir $hDir);
close $hDir;
if (scalar @stryFile == 1)
{
return;
}
sleep(1);
}
die "could not find $strDir/$strRegEx after $iSeconds second(s)";
}
sub pgbr_backup
{
my $strBackRestBinPath = shift;
my $strCluster = shift;
my $strCommand = "$strBackRestBinPath/pg_backrest.pl --config=$strBackRestBinPath/pg_backrest.conf backup $strCluster";
execute($strCommand);
}
my $strUser = execute('whoami');
my $strTestPath = "/Users/dsteele/test";
my $strDbDir = "db";
my $strArchiveDir = "backup/db/archive";
my $strBackupDir = "backup";
my $strPgBinPath = "/Library/PostgreSQL/9.3/bin";
my $strPort = "6001";
my $strBackRestBinPath = "/Users/dsteele/pg_backrest";
my $strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 0, 0);
################################################################################
# Stop the current test cluster if it is running and create a new one
################################################################################
eval {pg_stop($strPgBinPath, "$strTestPath/$strDbDir")};
if ($@)
{
print(" ... unable to stop pg server (ignoring): " . trim($@) . "\n\n")
}
pg_drop($strTestPath);
pg_create($strPgBinPath, $strTestPath, $strDbDir, $strArchiveDir, $strBackupDir);
pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand);
pg_password_set($strPgBinPath, "$strTestPath/$strDbDir/common", $strUser, $strPort);
################################################################################
# Connect and start tests
################################################################################
$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser,
'password', {AutoCommit => 1});
pg_execute($dbh, "create tablespace ts1 location '$strTestPath/$strDbDir/ts1'");
pg_execute($dbh, "create tablespace ts2 location '$strTestPath/$strDbDir/ts2'");
pg_execute($dbh, "create table test (id int)");
pg_execute($dbh, "create table test_ts1 (id int) tablespace ts1");
pg_execute($dbh, "create table test_ts2 (id int) tablespace ts1");
pg_execute($dbh, "insert into test values (1)");
pg_execute($dbh, "select pg_switch_xlog()");
execute("mkdir -p $strTestPath/$strArchiveDir/0000000100000000");
# Test for archive log file 000000010000000000000001
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000001\$", 5);
# Turn on log checksum for the next test
$dbh->disconnect();
pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common");
$strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 0, 1);
pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand);
$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser,
'password', {AutoCommit => 1});
# Write another value into the test table
pg_execute($dbh, "insert into test values (2)");
pg_execute($dbh, "select pg_switch_xlog()");
# Test for archive log file 000000010000000000000002
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000002-([a-f]|[0-9]){40}\$", 5);
# Turn on log compression and checksum for the next test
$dbh->disconnect();
pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common");
$strArchiveCommand = archive_command_build($strBackRestBinPath, "$strTestPath/$strArchiveDir", 1, 1);
pg_start($strPgBinPath, "$strTestPath/$strDbDir/common", $strPort, $strArchiveCommand);
$dbh = DBI->connect("dbi:Pg:dbname=postgres;port=$strPort;host=127.0.0.1", $strUser,
'password', {AutoCommit => 1});
# Write another value into the test table
pg_execute($dbh, "insert into test values (3)");
pg_execute($dbh, "select pg_switch_xlog()");
# Test for archive log file 000000010000000000000003
wait_for_file("$strTestPath/$strArchiveDir/0000000100000000", "^000000010000000000000003-([a-f]|[0-9]){40}\\.gz\$", 5);
$dbh->disconnect();
################################################################################
# Stop the server
################################################################################
#pg_stop($strPgBinPath, "$strTestPath/$strDbDir/common");
################################################################################
# Start an offline backup
################################################################################
#pgbr_backup($strBackRestBinPath, "db");