diff --git a/bin/pg_backrest_remote.pl b/bin/pg_backrest_remote.pl index 853695bdd..bc1b1ad22 100755 --- a/bin/pg_backrest_remote.pl +++ b/bin/pg_backrest_remote.pl @@ -113,7 +113,9 @@ while ($strCommand ne OP_EXIT) { $oFile->copy(PIPE_STDIN, undef, PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'), - undef, param_get(\%oParamHash, 'destination_compress')); + undef, param_get(\%oParamHash, 'destination_compress'), + undef, undef, undef, + param_get(\%oParamHash, 'destination_path_create')); $oRemote->output_write(); } @@ -151,7 +153,9 @@ while ($strCommand ne OP_EXIT) PATH_ABSOLUTE, param_get(\%oParamHash, 'destination_file'), param_get(\%oParamHash, 'source_compressed'), param_get(\%oParamHash, 'destination_compress'), - param_get(\%oParamHash, 'ignore_missing_source', false)) ? 'Y' : 'N'); + param_get(\%oParamHash, 'ignore_missing_source', false). + undef, undef, + param_get(\%oParamHash, 'destination_path_create')) ? 'Y' : 'N'); } elsif ($strCommand eq OP_FILE_MANIFEST) { diff --git a/lib/BackRest/Backup.pm b/lib/BackRest/Backup.pm index ff065f3f4..d9fd5cd74 100644 --- a/lib/BackRest/Backup.pm +++ b/lib/BackRest/Backup.pm @@ -153,10 +153,10 @@ sub backup_thread_complete $bConfessOnError = true; } - if (!defined($iTimeout)) - { - &log(WARN, "no thread timeout was set"); - } +# if (!defined($iTimeout)) +# { +# &log(WARN, "no thread timeout was set"); +# } # Wait for all threads to complete and handle errors my $iThreadComplete = 0; diff --git a/lib/BackRest/File.pm b/lib/BackRest/File.pm index fc716f2ea..f9caddff8 100644 --- a/lib/BackRest/File.pm +++ b/lib/BackRest/File.pm @@ -1119,9 +1119,9 @@ sub copy # Set debug string and log my $strDebug = ($bSourceRemote ? " remote" : " local") . " ${strSourcePathType}" . - (defined($strSourceOp) ? ":${strSourceFile}" : "") . + (defined($strSourceFile) ? ":${strSourceOp}" : "") . " to" . ($bDestinationRemote ? " remote" : " local") . " ${strDestinationPathType}" . - (defined($strDestinationOp) ? ":${strDestinationFile}" : "") . + (defined($strDestinationFile) ? ":${strDestinationOp}" : "") . ", source_compressed = " . ($bSourceCompressed ? "true" : "false") . ", destination_compress = " . ($bDestinationCompress ? "true" : "false") . ", ignore_missing_source = " . ($bIgnoreMissingSource ? "true" : "false") . @@ -1244,6 +1244,7 @@ sub copy { $oParamHash{destination_file} = $strDestinationOp; $oParamHash{destination_compress} = $bDestinationCompress; + $oParamHash{destination_path_create} = $bDestinationPathCreate; $hOut = $self->{oRemote}->{hIn}; } @@ -1257,6 +1258,7 @@ sub copy $oParamHash{source_compressed} = $bSourceCompressed; $oParamHash{destination_file} = $strDestinationOp; $oParamHash{destination_compress} = $bDestinationCompress; + $oParamHash{destination_path_create} = $bDestinationPathCreate; if ($bIgnoreMissingSource) { diff --git a/lib/BackRest/Remote.pm b/lib/BackRest/Remote.pm index 2b4d52fee..d312d5c1e 100644 --- a/lib/BackRest/Remote.pm +++ b/lib/BackRest/Remote.pm @@ -24,7 +24,7 @@ use BackRest::Utility; #################################################################################################################################### use constant { - DEFAULT_BLOCK_SIZE => 1048576 + DEFAULT_BLOCK_SIZE => 8192 #1048576 }; #################################################################################################################################### @@ -338,9 +338,12 @@ sub binary_xfer my $iBlockSize = $self->{iBlockSize}; my $iBlockIn; + my $iBlockInTotal = $iBlockSize; my $iBlockOut; + my $iBlockTotal = 0; my $strBlockHeader; my $strBlock; +# my $strBlockMore; my $oGzip; my $hPipeIn; my $hPipeOut; @@ -408,24 +411,52 @@ sub binary_xfer { if ($strRemote eq 'in') { - - $strBlockHeader = $self->read_line($hIn); - - if ($strBlockHeader !~ /^block [0-9]+$/) + if ($iBlockInTotal == $iBlockSize) { - confess "unable to read block header ${strBlockHeader}"; + $strBlockHeader = $self->read_line($hIn); + + if ($strBlockHeader !~ /^block [0-9]+$/) + { + $self->wait_pid(); + confess "unable to read block header ${strBlockHeader}"; + } + + $iBlockInTotal = 0; + $iBlockTotal += 1; } $iBlockSize = trim(substr($strBlockHeader, index($strBlockHeader, " ") + 1)); if ($iBlockSize != 0) { - $iBlockIn = sysread($hIn, $strBlock, $iBlockSize); +# print "looking for a block of size" - if (!defined($iBlockIn) || $iBlockIn != $iBlockSize) + $iBlockIn = sysread($hIn, $strBlock, $iBlockSize - $iBlockInTotal); + + # while (defined($iBlockIn) && $iBlockIn != $iBlockSize) + # { + # $iBlockInMore = sysread($hIn, $strBlockMore, $iBlockSize - $iBlockIn); + # + # confess "able to read $iBlockInMore bytes after reading $iBlockIn bytes"; + # + # if (!defined($iBlockInMore)) + # { + # $iBlockIn = undef; + # } + # + # $iBlockIn += $iBlockInMore; + # $strBlock += $strBlockMore; + # } + + if (!defined($iBlockIn)) { - confess "unable to read ${iBlockSize} bytes from remote" . (defined($!) ? ": " . $! : ""); + $self->wait_pid(); + confess "unable to read block #${iBlockTotal}/${iBlockSize} bytes from remote" . + (defined($iBlockIn) ? " (only ${iBlockIn} bytes read)" : " (nothing read)") . + (defined($!) ? ": " . $! : ""); } + + $iBlockInTotal += $iBlockIn; } else { @@ -438,6 +469,7 @@ sub binary_xfer if (!defined($iBlockIn)) { + $self->wait_pid(); confess &log(ERROR, "unable to read"); } } @@ -450,6 +482,7 @@ sub binary_xfer if (!defined($iBlockOut) || $iBlockOut != length($strBlockHeader)) { + $self->wait_pid(); confess "unable to write block header"; } } @@ -460,6 +493,7 @@ sub binary_xfer if (!defined($iBlockOut) || $iBlockOut != $iBlockIn) { + $self->wait_pid(); confess "unable to write ${iBlockIn} bytes" . (defined($!) ? ": " . $! : ""); } } diff --git a/test/lib/BackRestTest/BackupTest.pm b/test/lib/BackRestTest/BackupTest.pm index 4284d3070..6bfe675fd 100755 --- a/test/lib/BackRestTest/BackupTest.pm +++ b/test/lib/BackRestTest/BackupTest.pm @@ -129,8 +129,8 @@ sub BackRestTestBackup_Test BackRestTestBackup_Setup(); - BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB);#, REMOTE_BACKUP); - BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP);#, REMOTE_DB); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_DbPathGet() . '/pg_backrest.conf', REMOTE_DB, REMOTE_BACKUP); + BackRestTestCommon_ConfigCreate(BackRestTestCommon_BackupPathGet() . '/pg_backrest.conf', REMOTE_BACKUP, REMOTE_DB); BackRestTestCommon_Execute(BackRestTestCommon_CommandMainGet() . ' --config=' . BackRestTestCommon_BackupPathGet() . "/pg_backrest.conf --type=incr --stanza=${strStanza} backup"); diff --git a/test/lib/BackRestTest/CommonTest.pm b/test/lib/BackRestTest/CommonTest.pm index 6f8d7db1a..ff3bb577e 100755 --- a/test/lib/BackRestTest/CommonTest.pm +++ b/test/lib/BackRestTest/CommonTest.pm @@ -133,6 +133,7 @@ sub BackRestTestCommon_ConfigCreate $oParamHash{$strCommonStanza}{'path'} = $strCommonDbCommonPath; $oParamHash{'global:backup'}{'path'} = $strCommonBackupPath; + $oParamHash{'global:backup'}{'thread-max'} = '8'; $oParamHash{'global:log'}{'level-console'} = 'debug'; $oParamHash{'global:log'}{'level-file'} = 'trace';