db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2026-05-22 10:15:16 +02:00
Code Issues Releases Activity

Retry reads of pg_control until checksum is valid.

Browse Source 
On certain file systems (e.g. ext4) pg_control may appear torn if there is a concurrent write while reading the file. To prevent an invalid read, retry until the checksum matches the control data.

Special handling is required for the pg-version-force feature since the offset of the checksum is not known. In this case, scan from the default position to the end of the data looking for a checksum match. This is a bit imprecise, but better than nothing, and the chance of a random collision in the control data seems very remote considering the ratio of data size (< 512 bytes) to checksum size (4 bytes).

This was discovered and a possible solution proposed for PostgreSQL in [1]. The proposed solution may work for backup, but pgBackRest needs to be able to read pg_control reliably outside of backup. So no matter what fix is adopted for PostgreSQL, pgBackRest need retries. Further adjustment may be required as the PostgreSQL fix evolves.

[1] https://www.postgresql.org/message-id/20221123014224.xisi44byq3cf5psi%40awork3.anarazel.de
This commit is contained in:
David Steele
2023-09-10 09:47:49 -04:00
committed by GitHub
parent c1805134b3
commit f42d927d2d
21 changed files with 473 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/lib/pgBackRestTest/Env/HostEnvTest.pm
+79 -72
View File
@@ -271,48 +271,6 @@ sub dbCatalogVersion
return $hCatalogVersion->{$strPgVersion};
}
####################################################################################################################################
# Get database control version for the db version
####################################################################################################################################
sub dbControlVersion
{
my $self = shift;
# Assign function parameters, defaults, and log debug info
my
(
$strOperation,
$strPgVersion,
) =
logDebugParam
(
__PACKAGE__ . '->dbControlVersion', \@_,
{name => 'strPgVersion', trace => true},
);
my $hControlVersion =
{
&PG_VERSION_93 => 937,
&PG_VERSION_94 => 942,
&PG_VERSION_95 => 942,
&PG_VERSION_96 => 960,
&PG_VERSION_10 => 1002,
&PG_VERSION_11 => 1100,
&PG_VERSION_12 => 1201,
&PG_VERSION_13 => 1300,
&PG_VERSION_14 => 1300,
&PG_VERSION_15 => 1300,
&PG_VERSION_16 => 1300,
};
if (!defined($hControlVersion->{$strPgVersion}))
{
confess &log(ASSERT, "no control version defined for pg version ${strPgVersion}");
}
return $hControlVersion->{$strPgVersion};
}
####################################################################################################################################
# Generate control file content
####################################################################################################################################
@@ -332,48 +290,97 @@ sub controlGenerateContent
{name => 'strPgVersion', trace => true},
);
my $tControlContent = pack('Q', $self->dbSysId($strPgVersion));
$tControlContent .= pack('L', $self->dbControlVersion($strPgVersion));
$tControlContent .= pack('L', $self->dbCatalogVersion($strPgVersion));
# Offset to page size by architecture bits and version
my $rhOffsetToPageSize =
my $hControlContent =
{
32 =>
{
'9.3' => 180 - length($tControlContent),
'9.4' => 188 - length($tControlContent),
'9.5' => 200 - length($tControlContent),
'9.6' => 200 - length($tControlContent),
'10' => 200 - length($tControlContent),
'11' => 192 - length($tControlContent),
'12' => 196 - length($tControlContent),
'13' => 196 - length($tControlContent),
&PG_VERSION_93 =>
"5d0064a7b3b6e00da903000009b0ff0b00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"00000000000000000000000000200000000000000000000000000001000000000000000000000000000000000000000043d77e50",
&PG_VERSION_94 =>
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000002000000000000000000000000000010000000000000000000000000000000000000000" .
"000000006b0756c8",
&PG_VERSION_95 =>
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000003bfe413a",
&PG_VERSION_96 =>
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000005d135da6",
&PG_VERSION_10 =>
"640064a7b3b6e00dea030000cbce050c00000000000000000000000001000000000000000000000000000000000000000000000001000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000f8556c34",
},
64 =>
{
'9.3' => 192 - length($tControlContent),
'9.4' => 200 - length($tControlContent),
'9.5' => 216 - length($tControlContent),
'9.6' => 216 - length($tControlContent),
'10' => 216 - length($tControlContent),
'11' => 208 - length($tControlContent),
'12' => 212 - length($tControlContent),
'13' => 212 - length($tControlContent),
&PG_VERSION_93 =>
"5d0064a7b3b6e00da903000009b0ff0b00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000020000000000000000000000000000100000000000000000000000000000000" .
"0000000003fff8cc",
&PG_VERSION_94 =>
"5e0064a7b3b6e00dae0300000b43010c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000002000000000000000000000000000010000000000000000" .
"00000000000000000000000000000000ee6cf996",
&PG_VERSION_95 =>
"5f0064a7b3b6e00dae030000a3cc020c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"0000000000000001000000000000000000000000000000000000000000000000381ec2de",
&PG_VERSION_96 =>
"600064a7b3b6e00dc0030000c34b040c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"00000000000000010000000000000000000000000000000000000000000000002d96a4c0",
&PG_VERSION_10 =>
"640064a7b3b6e00dea030000cbce050c00000000000000000000000000000000010000000000000000000000000000000000000000000000" .
"0100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000" .
"0000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000" .
"00000000000000008d543cdf",
},
};
# Fill up to page size and set page size
$tControlContent .= ('C' x $rhOffsetToPageSize->{$self->archBits()}{$strPgVersion});
$tControlContent .= pack('L', 8192);
my $strControlContent = $hControlContent->{$self->archBits()}{$strPgVersion};
if (!defined($strControlContent))
{
confess &log(ASSERT, "no control content defined for pg version ${strPgVersion}");
}
my $tControlContent = '';
for (my $iIdx = 0; $iIdx < length($strControlContent) / 2; $iIdx++)
{
my $iChar = hex(substr($strControlContent, $iIdx * 2, 2));
$tControlContent .= pack('C', $iChar);
}
# Fill up to wal segment size and set wal segment size
$tControlContent .= ('C' x 8);
$tControlContent .= pack('L', 16 * 1024 * 1024);
# Pad bytes
$tControlContent .= ('C' x (8192 - length($tControlContent)));
for (my $iIdx = length($tControlContent); $iIdx < 8192; $iIdx++)
{
$tControlContent .= pack('C', 0);
}
return \$tControlContent;
}
test/lib/pgBackRestTest/Module/Mock/MockAllTest.pm
+1 -1
View File
@@ -372,7 +372,7 @@ sub run
$oHostDbPrimary->manifestFileCreate(
\%oManifest, MANIFEST_TARGET_PGDATA, DB_FILE_PGCONTROL, '[replaceme]',
$self->archBits() == 32 ? '8107e546c59c72a8c1818fc3610d7cc1e5623660' : '4c77c900f7af0d9ab13fa9982051a42e0b637f6c',
$self->archBits() == 32 ? '59331420592064afff045f352de3a1ce53b0dd21' : 'e8bd9ef18a0b58f89aed7e98a1c32119f03727ee',
$lTime - 100, undef, true);
# Copy pg_control