db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-12 14:56:08 +02:00
Code Issues Releases Activity

[Issue #169] pgFileGetCRC refactoring, now it operates only locally

Browse Source
This commit is contained in:
Grigory Smolkin 2020-03-03 16:57:09 +03:00
parent 8110715dbf
commit e9c31759c5
6 changed files with 23 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/archive.c
View File

@ -594,11 +594,11 @@ fileEqualCRC(const char *path1, const char *path2, bool path2_is_compressed)
else
#endif
{
crc2 = pgFileGetCRC(path2, true, true, NULL, FIO_BACKUP_HOST);
crc2 = fio_get_crc32(path2, FIO_BACKUP_HOST);
}
/* Get checksum of original file */
crc1 = pgFileGetCRC(path1, true, true, NULL, FIO_DB_HOST);
crc1 = fio_get_crc32(path1, FIO_DB_HOST);
return EQ_CRC32C(crc1, crc2);
}

23
src/backup.c
View File

@ -575,6 +575,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
{
parray *xlog_files_list;
char pg_xlog_path[MAXPGPATH];
char wal_full_path[MAXPGPATH];
/* Scan backup PG_XLOG_DIR */
xlog_files_list = parray_new();
@ -586,11 +587,13 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync)
for (i = 0; i < parray_num(xlog_files_list); i++)
{
pgFile *file = (pgFile *) parray_get(xlog_files_list, i);
join_path_components(wal_full_path, pg_xlog_path, file->rel_path);
if (S_ISREG(file->mode))
{
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->crc = pgFileGetCRC(wal_full_path, true, false);
file->write_size = file->size;
}
/* Remove file path root prefix*/
if (strstr(file->path, database_path) == file->path)
@ -1805,10 +1808,11 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
{
file = pgFileNew(backup_label, PG_BACKUP_LABEL_FILE, true, 0,
FIO_BACKUP_HOST);
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->uncompressed_size = file->read_size;
file->crc = pgFileGetCRC(backup_label, true, false);
file->write_size = file->size;
file->uncompressed_size = file->size;
free(file->path);
file->path = strdup(PG_BACKUP_LABEL_FILE);
parray_append(backup_files_list, file);
@ -1854,9 +1858,8 @@ pg_stop_backup(pgBackup *backup, PGconn *pg_startbackup_conn,
FIO_BACKUP_HOST);
if (S_ISREG(file->mode))
{
file->crc = pgFileGetCRC(file->path, true, false,
&file->read_size, FIO_BACKUP_HOST);
file->write_size = file->read_size;
file->crc = pgFileGetCRC(tablespace_map, true, false);
file->write_size = file->size;
}
free(file->path);
file->path = strdup(PG_TABLESPACE_MAP_FILE);

64
src/dir.c
View File

@ -259,10 +259,9 @@ delete_file:
* We cannot make decision about file decompression because
* user may ask to backup already compressed files and we should be
* obvious about it.
* TODO: add decompression option.
*/
pg_crc32
pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok)
{
FILE *fp;
pg_crc32 crc = 0;
@ -316,65 +315,6 @@ pgFileGetCRCnew(const char *file_path, bool use_crc32c, bool missing_ok)
return crc;
}
/*
* Read the file to compute its CRC.
* As a handy side effect, we return filesize via bytes_read parameter.
*/
pg_crc32
pgFileGetCRC(const char *file_path, bool use_crc32c, bool raise_on_deleted,
size_t *bytes_read, fio_location location)
{
FILE *fp;
pg_crc32 crc = 0;
char buf[STDIO_BUFSIZE];
size_t len = 0;
size_t total = 0;
int errno_tmp;
INIT_FILE_CRC32(use_crc32c, crc);
/* open file in binary read mode */
fp = fio_fopen(file_path, PG_BINARY_R, location);
if (fp == NULL)
{
if (!raise_on_deleted && errno == ENOENT)
{
FIN_FILE_CRC32(use_crc32c, crc);
return crc;
}
else
elog(ERROR, "cannot open file \"%s\": %s",
file_path, strerror(errno));
}
/* calc CRC of file */
for (;;)
{
if (interrupted)
elog(ERROR, "interrupted during CRC calculation");
len = fio_fread(fp, buf, sizeof(buf));
if (len == 0)
break;
/* update CRC */
COMP_FILE_CRC32(use_crc32c, crc, buf, len);
total += len;
}
if (bytes_read)
*bytes_read = total;
errno_tmp = errno;
if (len < 0)
elog(WARNING, "cannot read \"%s\": %s", file_path,
strerror(errno_tmp));
FIN_FILE_CRC32(use_crc32c, crc);
fio_fclose(fp);
return crc;
}
void
pgFileFree(void *file)
{
@ -1775,7 +1715,7 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
FIO_BACKUP_HOST);
pfree(file->path);
file->path = pgut_strdup(DATABASE_MAP);
file->crc = pgFileGetCRCnew(database_map_path, true, false);
file->crc = pgFileGetCRC(database_map_path, true, false);
file->write_size = file->read_size;
file->uncompressed_size = file->read_size;

7
src/pg_probackup.h
View File

@ -826,10 +826,9 @@ extern pgFile *pgFileInit(const char *path, const char *rel_path);
extern void pgFileDelete(pgFile *file, const char *full_path);
extern void pgFileFree(void *file);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c,
bool raise_on_deleted, size_t *bytes_read, fio_location location);
extern pg_crc32 pgFileGetCRCnew(const char *file_path, bool missing_ok, bool use_crc32c);
//extern pg_crc32 pgFileGetCRC_compressed(const char *file_path, bool use_crc32c, bool missing_ok);
extern pg_crc32 pgFileGetCRC(const char *file_path, bool missing_ok, bool use_crc32c);
extern int pgFileCompareName(const void *f1, const void *f2);
extern int pgFileComparePath(const void *f1, const void *f2);
extern int pgFileMapComparePath(const void *f1, const void *f2);

4
src/utils/file.c
View File

@ -926,7 +926,7 @@ pg_crc32 fio_get_crc32(const char *file_path, fio_location location)
return crc;
}
else
return pgFileGetCRCnew(file_path, true, true);
return pgFileGetCRC(file_path, true, true);
}
/* Remove file */
@ -1629,7 +1629,7 @@ void fio_communicate(int in, int out)
break;
case FIO_GET_CRC32:
/* calculate crc32 for a file */
crc = pgFileGetCRCnew(buf, true, true);
crc = pgFileGetCRC(buf, true, true);
IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc));
break;
default:

2
src/validate.c
View File

@ -328,7 +328,7 @@ pgBackupValidateFiles(void *arg)
crc = pgFileGetCRC(file->path,
arguments->backup_version <= 20021 ||
arguments->backup_version >= 20025,
true, NULL, FIO_LOCAL_HOST);
false);
if (crc != file->crc)
{
elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X",