Added file and directory syncs to the File object for additional safety during backup and archiving.

README.md

@@ -771,6 +771,8 @@ example: db-path=/data/db
 
 * Removed pg_backrest_remote and added the functionality to pg_backrest as remote command.
 
+* Added file and directory syncs to the File object for additional safety during backup and archiving.  Suggested by Andres Freund.
+
 ### v0.75: New repository format, info command and experimental 9.5 support
 
 * IMPORTANT NOTE: This flag day release breaks compatibility with older versions of PgBackRest.  The manifest format, on-disk structure, and the binary names have all changed.  You must create a new repository to hold backups for this version of PgBackRest and keep your older repository for a time in case you need to do a restore.  The `pg_backrest.conf` file has not changed but you'll need to change any references to `pg_backrest.pl` in cron (or elsewhere) to `pg_backrest` (without the `.pl` extension).

doc/doc.xml

@@ -727,6 +727,9 @@ Run a <id>full</id> backup on the <id>db</id> stanza.  <param>--type</param> can
 					<release-feature>
 						<text>Removed pg_backrest_remote and added the functionality to pg_backrest as remote command.</text>
 					</release-feature>
+					<release-feature>
+						<text>Added file and directory syncs to the File object for additional safety during backup and archiving.  Suggested by Andres Freund.</text>
+					</release-feature>
 				</release-feature-bullet-list>
 			</release-version>
 

lib/BackRest/Exception.pm

@@ -41,6 +41,9 @@ use constant
     ERROR_HOST_CONNECT                 => 124,
     ERROR_LOCK_ACQUIRE                 => 125,
     ERROR_BACKUP_MISMATCH              => 126,
+    ERROR_FILE_SYNC                    => 127,
+    ERROR_PATH_OPEN                    => 128,
+    ERROR_PATH_SYNC                    => 129,
 
     ERROR_UNKNOWN                      => 199
 };
@@ -50,7 +53,7 @@ our @EXPORT = qw(ERROR_ASSERT ERROR_CHECKSUM ERROR_CONFIG ERROR_FILE_INVALID ERR
                  ERROR_OPTION_DUPLICATE_KEY ERROR_OPTION_NEGATE ERROR_OPTION_REQUIRED ERROR_POSTMASTER_RUNNING ERROR_PROTOCOL
                  ERROR_RESTORE_PATH_NOT_EMPTY ERROR_FILE_OPEN ERROR_FILE_READ ERROR_PARAM_REQUIRED ERROR_ARCHIVE_MISMATCH
                  ERROR_ARCHIVE_DUPLICATE ERROR_VERSION_NOT_SUPPORTED ERROR_PATH_CREATE ERROR_OPERATION_INVALID ERROR_HOST_CONNECT
-                 ERROR_UNKNOWN ERROR_LOCK_ACQUIRE ERROR_BACKUP_MISMATCH);
+                 ERROR_UNKNOWN ERROR_LOCK_ACQUIRE ERROR_BACKUP_MISMATCH ERROR_FILE_SYNC ERROR_PATH_OPEN ERROR_PATH_SYNC);
 
 ####################################################################################################################################
 # CONSTRUCTOR

lib/BackRest/File.pm

@@ -14,11 +14,13 @@ use File::Basename qw(dirname basename);
 use File::Copy qw(cp);
 use File::Path qw(make_path remove_tree);
 use File::stat;
+use IO::Handle;
 use Net::OpenSSH;
 
 use lib dirname($0) . '/../lib';
 use BackRest::Config;
 use BackRest::Exception;
+use BackRest::FileCommon;
 use BackRest::Protocol;
 use BackRest::Utility;
 
@@ -76,19 +78,21 @@ push @EXPORT, qw(PIPE_STDIN PIPE_STDOUT PIPE_STDERR);
 ####################################################################################################################################
 use constant
 {
-    OP_FILE_OWNER       => 'File->owner',
+    OP_FILE_OWNER               => 'File->owner',
-    OP_FILE_WAIT        => 'File->wait',
+    OP_FILE_WAIT                => 'File->wait',
-    OP_FILE_LIST        => 'File->list',
+    OP_FILE_LIST                => 'File->list',
-    OP_FILE_EXISTS      => 'File->exists',
+    OP_FILE_EXISTS              => 'File->exists',
-    OP_FILE_HASH        => 'File->hash',
+    OP_FILE_HASH                => 'File->hash',
-    OP_FILE_REMOVE      => 'File->remove',
+    OP_FILE_REMOVE              => 'File->remove',
-    OP_FILE_MANIFEST    => 'File->manifest',
+    OP_FILE_MANIFEST            => 'File->manifest',
-    OP_FILE_COMPRESS    => 'File->compress',
+    OP_FILE_COMPRESS            => 'File->compress',
-    OP_FILE_MOVE        => 'File->move',
+    OP_FILE_MOVE                => 'File->move',
-    OP_FILE_COPY        => 'File->copy',
+    OP_FILE_COPY                => 'File->copy',
-    OP_FILE_COPY_OUT    => 'File->copy_out',
+    OP_FILE_COPY_OUT            => 'File->copy_out',
-    OP_FILE_COPY_IN     => 'File->copy_in',
+    OP_FILE_COPY_IN             => 'File->copy_in',
-    OP_FILE_PATH_CREATE => 'File->path_create',
+    OP_FILE_PATH_CREATE         => 'File->path_create',
+    OP_FILE_PATH_SYNC           => 'File->pathSync',
+    OP_FILE_PATH_SYNC_STATIC    => 'File::filePathSync',
     OP_FILE_LINK_CREATE => 'File->link_create'
 };
 
@@ -454,6 +458,22 @@ sub link_create
     }
 }
 
+####################################################################################################################################
+# pathSync
+#
+# Sync a directory.
+####################################################################################################################################
+sub pathSync
+{
+    my $self = shift;
+    my $strPathType = shift;
+    my $strPath = shift;
+
+    &log(TRACE, OP_FILE_PATH_SYNC . "(): pathType = ${strPathType}, path = ${strPath}");
+    
+    filePathSync($self->path_get($strPathType, $strPath eq '.' ? undef : $strPath));
+}
+
 ####################################################################################################################################
 # MOVE
 #
@@ -526,6 +546,8 @@ sub move
                 }
             }
         }
+        
+        $self->pathSync($strDestinationPathType, dirname($strDestinationFile));
     }
 }
 
@@ -1662,16 +1684,20 @@ sub copy
         }
     }
 
-    # Close the source file (if open)
+    # Close the source file (if local)
     if (defined($hSourceFile))
     {
         close($hSourceFile) or confess &log(ERROR, "cannot close file ${strSourceOp}");
     }
 
-    # Close the destination file (if open)
+    # Sync and close the destination file (if local)
     if (defined($hDestinationFile))
     {
-        close($hDestinationFile) or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
+        $hDestinationFile->sync()
+            or confess &log(ERROR, "unable to sync ${strDestinationTmpOp}", ERROR_FILE_SYNC);
+
+        close($hDestinationFile)
+            or confess &log(ERROR, "cannot close file ${strDestinationTmpOp}");
     }
 
     # Checksum and file size should be set if the destination is not remote

lib/BackRest/FileCommon.pm

@@ -0,0 +1,47 @@
+####################################################################################################################################
+# FILE COMMON    MODULE
+####################################################################################################################################
+package BackRest::FileCommon;
+
+use strict;
+use warnings FATAL => qw(all);
+use Carp qw(confess);
+
+use Exporter qw(import);
+use File::Basename qw(dirname);
+use IO::Handle;
+
+use lib dirname($0) . '/../lib';
+use BackRest::Exception;
+use BackRest::Utility;
+
+####################################################################################################################################
+# Operation constants
+####################################################################################################################################
+use constant OP_FILE_BASE                                         => 'FileBase';
+
+use constant OP_FILE_BASE_PATH_SYNC                               => OP_FILE_BASE . '::filePathSync';
+
+####################################################################################################################################
+# filePathSync
+#
+# Sync a directory.
+####################################################################################################################################
+sub filePathSync
+{
+    my $strPath = shift;
+
+    &log(TRACE, OP_FILE_BASE_PATH_SYNC . "(): path = ${strPath}");
+    
+    open(my $hPath, "<", $strPath)
+        or confess &log(ERROR, "unable to open ${strPath}", ERROR_PATH_OPEN);
+    open(my $hPathDup, ">&", $hPath)
+        or confess &log(ERROR, "unable to duplicate handle for ${strPath}", ERROR_PATH_OPEN);
+
+    $hPathDup->sync
+        or confess &log(ERROR, "unable to sync ${strPath}", ERROR_PATH_SYNC);
+}
+
+our @EXPORT = qw(filePathSync);
+
+1;

lib/BackRest/Ini.pm

@@ -9,11 +9,13 @@ use Carp qw(confess);
 
 use Exporter qw(import);
 use File::Basename qw(dirname basename);
+use IO::Handle;
 use JSON::PP;
 use Storable qw(dclone);
 
 use lib dirname($0);
 use BackRest::Exception;
+use BackRest::FileCommon;
 use BackRest::Utility;
 
 ####################################################################################################################################
@@ -272,6 +274,9 @@ sub iniSave
         $bFirst = false;
     }
 
+    # Sync and close ini file
+    $hFile->sync();
+    filePathSync(dirname($strFileName));
     close($hFile);
 }