1
0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00

[PBCKP-345] rework file stat handling to simplify and cross-platform-ability

- don't use full `struct stat` since it differs between platforms
- store file kind separately from mode in pb specific way
- add pioFilesAreSame as replacement of fio_is_same_file since it needs
  st_ino and st_dev, therefore had to run on localdrive always.
This commit is contained in:
Yura Sokolov
2022-11-16 03:45:32 +03:00
parent af806800e9
commit e9055001ac
14 changed files with 356 additions and 176 deletions
+3 -3
View File
@@ -388,7 +388,7 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
char to_fullpath[MAXPGPATH];
char to_fullpath_part[MAXPGPATH];
/* partial handling */
struct stat st;
pio_stat_t st;
int partial_try_count = 0;
ssize_t partial_file_size = 0;
bool partial_is_stale = true;
@@ -466,11 +466,11 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
elog(LOG,
"Temp WAL file already exists, waiting on it %u seconds: \"%s\"",
archive_timeout, to_fullpath_part);
partial_file_size = st.st_size;
partial_file_size = st.pst_size;
}
/* file size is changing */
if (st.st_size != partial_file_size)
if (st.pst_size != partial_file_size)
partial_is_stale = false;
sleep(1);
+11 -11
View File
@@ -392,7 +392,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
/* if the entry was a directory, create it in the backup */
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
{
char dirpath[MAXPGPATH];
@@ -569,7 +569,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
/* TODO: sync directory ? */
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
continue;
if (file->write_size <= 0)
@@ -1837,7 +1837,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c
file = pgFileNew(full_filename, filename, true, 0,
FIO_BACKUP_HOST);
if (S_ISREG(file->mode))
if (file->kind == PIO_KIND_REGULAR)
{
file->crc = pgFileGetCRC32C(full_filename, false);
@@ -1991,7 +1991,7 @@ backup_files(void *arg)
pgFile *prev_file = NULL;
/* We have already copied all directories */
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
continue;
if (arguments->thread_num == 1)
@@ -2046,9 +2046,9 @@ backup_files(void *arg)
}
/* Encountered some strange beast */
if (!S_ISREG(file->mode))
elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
file->mode, from_fullpath);
if (file->kind != PIO_KIND_REGULAR)
elog(WARNING, "Unexpected type %s of file \"%s\", skipping",
pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
/* Check that file exist in previous backup */
if (current.backup_mode != BACKUP_MODE_FULL)
@@ -2121,7 +2121,7 @@ parse_filelist_filenames(parray *files, const char *root)
pgFile *file = (pgFile *) parray_get(files, i);
int sscanf_result;
if (S_ISREG(file->mode) &&
if (file->kind == PIO_KIND_REGULAR &&
path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
{
/*
@@ -2148,7 +2148,7 @@ parse_filelist_filenames(parray *files, const char *root)
}
}
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 &&
file->name && file->name[0])
{
if (file->forkName == init)
@@ -2218,7 +2218,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL)
{
if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
if (prev_file->kind == PIO_KIND_REGULAR && prev_file->is_datafile)
{
elog(LOG, "Setting 'is_cfs' on file %s, name %s",
prev_file->rel_path, prev_file->name);
@@ -2375,7 +2375,7 @@ calculate_datasize_of_filelist(parray *filelist)
if (file->external_dir_num != 0 || file->excluded)
continue;
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
{
// TODO is a dir always 4K?
bytes += 4096;
+25 -7
View File
@@ -847,12 +847,22 @@ pgBackupGetBackupMode(pgBackup *backup, bool show_color)
static bool
IsDir(const char *dirpath, const char *entry, fio_location location)
{
FOBJ_FUNC_ARP();
char path[MAXPGPATH];
struct stat st;
pio_stat_t st;
err_i err;
join_path_components(path, dirpath, entry);
return fio_stat(location, path, &st, false) == 0 && S_ISDIR(st.st_mode);
st = $i(pioStat, pioDriveForLocation(location),
.path = path, .follow_symlink = false, .err = &err);
if ($haserr(err))
{
ft_logerr(FT_WARNING, $errmsg(err), "IsDir");
return false;
}
return st.pst_kind == PIO_KIND_DIRECTORY;
}
/*
@@ -1074,6 +1084,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
char path[MAXPGPATH];
char linked[MAXPGPATH];
char compress_alg_string[MAXPGPATH];
char kind[16];
int64 write_size,
uncompressed_size,
mode, /* bit length of mode_t depends on platforms */
@@ -1115,6 +1126,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
/*
* Optional fields
*/
if (get_control_value_str(buf, "kind", kind, sizeof(kind), false))
file->kind = pio_str2file_kind(kind, path);
else /* fallback to mode for old backups */
file->kind = pio_statmode2file_kind(file->mode, path);
if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0])
{
file->linked = pgut_strdup(linked);
@@ -1146,7 +1162,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
if (!file->is_datafile || file->is_cfs)
file->size = file->uncompressed_size;
if (file->external_dir_num == 0 && S_ISREG(file->mode))
if (file->external_dir_num == 0 && file->kind == PIO_KIND_REGULAR)
{
bool is_datafile = file->is_datafile;
set_forkname(file);
@@ -2564,14 +2580,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
if (file->write_size == FILE_NOT_FOUND)
continue;
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
{
backup_size_on_disk += 4096;
uncompressed_size_on_disk += 4096;
}
/* Count the amount of the data actually copied */
if (S_ISREG(file->mode) && file->write_size > 0)
if (file->kind == PIO_KIND_REGULAR && file->write_size > 0)
{
/*
* Size of WAL files in 'pg_wal' is counted separately
@@ -2587,11 +2603,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
}
len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"kind\":\"%s\", \"mode\":\"%u\", \"is_datafile\":\"%u\", "
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
"\"dbOid\":\"%u\"",
file->rel_path, file->write_size, file->mode,
file->rel_path, file->write_size,
pio_file_kind2str(file->kind, file->rel_path),
file->mode,
file->is_datafile ? 1 : 0,
file->is_cfs ? 1 : 0,
file->crc,
+6 -6
View File
@@ -378,7 +378,7 @@ catchup_thread_runner(void *arg)
pgFile *dest_file = NULL;
/* We have already copied all directories */
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
continue;
if (file->excluded)
@@ -400,9 +400,9 @@ catchup_thread_runner(void *arg)
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
/* Encountered some strange beast */
if (!S_ISREG(file->mode))
elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
file->mode, from_fullpath);
if (file->kind != PIO_KIND_REGULAR)
elog(WARNING, "Unexpected kind %s of file \"%s\", skipping",
pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
/* Check that file exist in dest pgdata */
if (arguments->backup_mode != BACKUP_MODE_FULL)
@@ -546,7 +546,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p
* - but PG itself is not relying on fs, its durable_sync
* includes directory sync
*/
if (S_ISDIR(file->mode) || file->excluded)
if (file->kind == PIO_KIND_DIRECTORY || file->excluded)
continue;
Assert(file->external_dir_num == 0);
@@ -807,7 +807,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
pgFile *file = (pgFile *) parray_get(source_filelist, i);
char parent_dir[MAXPGPATH];
if (!S_ISDIR(file->mode) || file->excluded)
if (file->kind != PIO_KIND_DIRECTORY || file->excluded)
continue;
/*
+2 -2
View File
@@ -147,7 +147,7 @@ check_files(void *arg)
elog(ERROR, "interrupted during checkdb");
/* No need to check directories */
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
continue;
if (!pg_atomic_test_set_flag(&file->lock))
@@ -161,7 +161,7 @@ check_files(void *arg)
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
i + 1, n_files_list, from_fullpath);
if (S_ISREG(file->mode))
if (file->kind == PIO_KIND_REGULAR)
{
/* check only uncompressed by cfs datafiles */
if (file->is_datafile && !file->is_cfs)
+30 -28
View File
@@ -145,23 +145,26 @@ pgFile *
pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
int external_dir_num, fio_location location)
{
struct stat st;
FOBJ_FUNC_ARP();
pio_stat_t st;
pgFile *file;
err_i err;
/* stat the file */
if (fio_stat(location, path, &st, follow_symlink) < 0)
{
st = $i(pioStat, pioDriveForLocation(location), .path = path,
.follow_symlink = follow_symlink, .err = &err);
if ($haserr(err)) {
/* file not found is not an error case */
if (errno == ENOENT)
if (getErrno(err) == ENOENT)
return NULL;
elog(ERROR, "cannot stat file \"%s\": %s", path,
strerror(errno));
ft_logerr(FT_FATAL, $errmsg(err), "pgFileNew");
}
file = pgFileInit(rel_path);
file->size = st.st_size;
file->mode = st.st_mode;
file->mtime = st.st_mtime;
file->size = st.pst_size;
file->kind = st.pst_kind;
file->mode = st.pst_mode;
file->mtime = st.pst_mtime;
file->external_dir_num = external_dir_num;
return file;
@@ -393,7 +396,7 @@ dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool fol
return;
}
if (!S_ISDIR(file->mode))
if (file->kind != PIO_KIND_DIRECTORY)
{
if (external_dir_num > 0)
elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected",
@@ -435,7 +438,7 @@ dir_check_file(pgFile *file, bool backup_logs)
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path);
/* Check if we need to exclude file by name */
if (S_ISREG(file->mode))
if (file->kind == PIO_KIND_REGULAR)
{
for (i = 0; pgdata_exclude_files[i]; i++)
if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0)
@@ -449,7 +452,7 @@ dir_check_file(pgFile *file, bool backup_logs)
* If the directory name is in the exclude list, do not list the
* contents.
*/
else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0)
else if (file->kind == PIO_KIND_DIRECTORY && !in_tablespace && file->external_dir_num == 0)
{
/*
* If the item in the exclude list starts with '/', compare to
@@ -478,14 +481,14 @@ dir_check_file(pgFile *file, bool backup_logs)
}
/* Do not backup ptrack_init files */
if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0)
if (file->kind == PIO_KIND_REGULAR && strcmp(file->name, "ptrack_init") == 0)
return CHECK_FALSE;
/*
* Check files located inside database directories including directory
* 'global'
*/
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 &&
file->name && file->name[0])
{
if (strcmp(file->name, "pg_internal.init") == 0)
@@ -607,7 +610,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
struct dirent *dent;
bool in_tablespace = false;
if (!S_ISDIR(parent->mode))
if (parent->kind != PIO_KIND_DIRECTORY)
elog(ERROR, "\"%s\" is not a directory", parent_dir);
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path);
@@ -641,7 +644,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
continue;
/* Skip entries point current dir or parent dir */
if (S_ISDIR(file->mode) &&
if (file->kind == PIO_KIND_DIRECTORY &&
(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
{
pgFileFree(file);
@@ -659,7 +662,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (!S_ISDIR(file->mode) && !S_ISREG(file->mode))
if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR)
{
elog(WARNING, "Skip '%s': unexpected file format", child);
pgFileFree(file);
@@ -671,7 +674,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
* Do not copy tablespaces twice. It may happen if the tablespace is located
* inside the PGDATA.
*/
if (S_ISDIR(file->mode) &&
if (file->kind == PIO_KIND_DIRECTORY &&
strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
{
Oid tblspcOid;
@@ -718,7 +721,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
* If the entry is a directory call dir_list_file_internal()
* recursively.
*/
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink,
backup_logs, skip_hidden, external_dir_num, location);
}
@@ -886,7 +889,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
{
pgFile *file = (pgFile *) parray_get(dest_files, i);
if (!S_ISDIR(file->mode))
if (file->kind != PIO_KIND_DIRECTORY)
continue;
/* skip external directory content */
@@ -919,7 +922,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
char parent_dir[MAXPGPATH];
pgFile *dir = (pgFile *) parray_get(dest_files, i);
if (!S_ISDIR(dir->mode))
if (dir->kind != PIO_KIND_DIRECTORY)
continue;
/* skip external directory content */
@@ -1523,14 +1526,13 @@ dir_is_empty(const char *path, fio_location location)
bool
fileExists(const char *path, fio_location location)
{
struct stat buf;
FOBJ_FUNC_ARP();
err_i err;
bool exists;
if (fio_stat(location, path, &buf, true) == -1 && errno == ENOENT)
return false;
else if (!S_ISREG(buf.st_mode))
return false;
else
return true;
exists = $i(pioExists, pioDriveForLocation(location), path, &err);
return exists;
}
/*
+12 -6
View File
@@ -28,9 +28,10 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t *
{
int fd;
char *buffer;
struct stat statbuf;
pio_stat_t statbuf;
char fullpath[MAXPGPATH];
int len;
size_t len;
err_i err;
join_path_components(fullpath, datadir, path);
@@ -43,16 +44,21 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t *
fullpath, strerror(errno));
}
if (fio_stat(location, fullpath, &statbuf, true) < 0)
statbuf = $i(pioStat, pioDriveForLocation(location), .path = fullpath,
.follow_symlink = true, .err = &err);
if ($haserr(err))
{
if (safe)
return NULL;
else
elog(ERROR, "Could not stat file \"%s\": %s",
fullpath, strerror(errno));
ft_logerr(FT_FATAL, $errmsg(err), "slurpFile");
}
len = statbuf.st_size;
if (statbuf.pst_size > SIZE_MAX)
ft_log(FT_FATAL, "file \"%s\" is too large: %lld",
fullpath, (long long)statbuf.pst_size);
len = statbuf.pst_size;
buffer = pg_malloc(len + 1);
if (fio_read(fd, buffer, len) != len)
+3 -2
View File
@@ -639,7 +639,7 @@ merge_chain(InstanceState *instanceState,
pgFile *file = (pgFile *) parray_get(dest_backup->files, i);
/* if the entry was an external directory, create it in the backup */
if (file->external_dir_num && S_ISDIR(file->mode))
if (file->external_dir_num && file->kind == PIO_KIND_DIRECTORY)
{
char dirpath[MAXPGPATH];
char new_container[MAXPGPATH];
@@ -955,6 +955,7 @@ merge_files(void *arg)
continue;
tmp_file = pgFileInit(dest_file->rel_path);
tmp_file->kind = dest_file->kind;
tmp_file->mode = dest_file->mode;
tmp_file->is_datafile = dest_file->is_datafile;
tmp_file->is_cfs = dest_file->is_cfs;
@@ -962,7 +963,7 @@ merge_files(void *arg)
tmp_file->dbOid = dest_file->dbOid;
/* Directories were created before */
if (S_ISDIR(dest_file->mode))
if (dest_file->kind == PIO_KIND_DIRECTORY)
goto done;
elog(progress ? INFO : LOG, "Progress: (%d/%zu). Merging file \"%s\"",
+13 -7
View File
@@ -291,6 +291,7 @@ main(int argc, char *argv[])
{
char *command = NULL;
ProbackupSubcmd backup_subcmd = NO_CMD;
err_i err;
ft_init_log(elog_ft_log);
fobj_init();
@@ -536,13 +537,17 @@ main(int argc, char *argv[])
if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD &&
backup_subcmd != ARCHIVE_GET_CMD)
{
struct stat st;
pio_stat_t st;
if (fio_stat(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path,
&st, true) != 0)
st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST),
.path = instanceState->instance_backup_subdir_path,
.follow_symlink = true,
.err = &err);
if ($haserr(err))
{
elog(WARNING, "Failed to access directory \"%s\": %s",
instanceState->instance_backup_subdir_path, strerror(errno));
ft_logerr(FT_WARNING, $errmsg(err), "Failed to access directory \"%s\"",
instanceState->instance_backup_subdir_path);
// TODO: redundant message, should we get rid of it?
elog(ERROR, "Instance '%s' does not exist in this backup catalog",
@@ -551,7 +556,7 @@ main(int argc, char *argv[])
else
{
/* Ensure that backup_path is a path to a directory */
if (!S_ISDIR(st.st_mode))
if (st.pst_kind != PIO_KIND_DIRECTORY)
elog(ERROR, "-B, --backup-path must be a path to directory");
}
}
@@ -896,7 +901,8 @@ main(int argc, char *argv[])
*/
char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
if (fio_is_same_file(FIO_DB_HOST, stripped_wal_file_path, archive_push_xlog_dir, true))
if ($i(pioFilesAreSame, pioDriveForLocation(FIO_DB_HOST),
.file1 = stripped_wal_file_path, .file2 = archive_push_xlog_dir))
{
/* 2nd case */
system_id = get_system_identifier(FIO_DB_HOST, instance_config.pgdata, false);
+10 -8
View File
@@ -211,18 +211,21 @@ typedef enum ForkName
typedef struct pgFile
{
char *name; /* file or directory name */
mode_t mode; /* protection (file type and permission) */
size_t size; /* size of the file */
time_t mtime; /* file st_mtime attribute, can be used only
during backup */
size_t read_size; /* size of the portion read (if only some pages are
pio_file_kind_e kind; /* kind of file */
uint32_t mode; /* protection (permission) */
uint64_t size; /* size of the file */
uint64_t read_size; /* size of the portion read (if only some pages are
backed up, it's different from size) */
int64 write_size; /* size of the backed-up file. BYTES_INVALID means
int64_t write_size; /* size of the backed-up file. BYTES_INVALID means
that the file existed but was not backed up
because not modified since last backup. */
size_t uncompressed_size; /* size of the backed-up file before compression
uint64_t uncompressed_size; /* size of the backed-up file before compression
* and adding block headers.
*/
time_t mtime; /* file st_mtime attribute, can be used only
during backup */
/* we need int64 here to store '-1' value */
pg_crc32 crc; /* CRC value of the file, regular file only */
char *rel_path; /* relative path of the file */
@@ -1022,7 +1025,6 @@ extern bool backup_contains_external(const char *dir, parray *dirs_list);
extern bool dir_is_empty(const char *path, fio_location location);
extern bool fileExists(const char *path, fio_location location);
extern size_t pgFileSize(const char *path);
extern pgFile *pgFileNew(const char *path, const char *rel_path,
bool follow_symlink, int external_dir_num,
+13 -13
View File
@@ -837,11 +837,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
{
pgFile *file = (pgFile *) parray_get(dest_files, i);
if (S_ISDIR(file->mode))
if (file->kind == PIO_KIND_DIRECTORY)
total_bytes += 4096;
if (!params->skip_external_dirs &&
file->external_dir_num && S_ISDIR(file->mode))
file->external_dir_num && file->kind == PIO_KIND_DIRECTORY)
{
char *external_path;
char dirpath[MAXPGPATH];
@@ -926,7 +926,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
redundant = true;
/* do not delete the useful internal directories */
if (S_ISDIR(file->mode) && !redundant)
if (file->kind == PIO_KIND_DIRECTORY && !redundant)
continue;
/* if file does not exists in destination list, then we can safely unlink it */
@@ -1056,7 +1056,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
char to_fullpath[MAXPGPATH];
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
if (S_ISDIR(dest_file->mode))
if (dest_file->kind == PIO_KIND_DIRECTORY)
continue;
/* skip external files if ordered to do so */
@@ -1137,7 +1137,7 @@ restore_files(void *arg)
pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i);
/* Directories were created before */
if (S_ISDIR(dest_file->mode))
if (dest_file->kind == PIO_KIND_DIRECTORY)
continue;
if (!pg_atomic_test_set_flag(&dest_file->lock))
@@ -1543,13 +1543,14 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup,
char path[MAXPGPATH];
FILE *fp = NULL;
FILE *fp_tmp = NULL;
struct stat st;
pio_stat_t st;
char current_time_str[100];
/* postgresql.auto.conf parsing */
char line[16384] = "\0";
char *buf = NULL;
int buf_len = 0;
int buf_len_max = 16384;
err_i err;
elog(LOG, "update recovery settings in postgresql.auto.conf");
@@ -1557,17 +1558,16 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup,
join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf");
if (fio_stat(FIO_DB_HOST, postgres_auto_path, &st, false) < 0)
st = $i(pioStat, pioDriveForLocation(FIO_DB_HOST),
.path = postgres_auto_path, .follow_symlink = false, .err = &err);
/* file not found is not an error case */
if ($haserr(err) && getErrno(err) != ENOENT)
{
/* file not found is not an error case */
if (errno != ENOENT)
elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path,
strerror(errno));
st.st_size = 0;
ft_logerr(FT_FATAL, $errmsg(err), "");
}
/* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */
if (st.st_size > 0)
if (st.pst_size > 0)
{
fp = fio_open_stream(FIO_DB_HOST, postgres_auto_path);
if (fp == NULL)
+190 -69
View File
@@ -51,6 +51,7 @@ typedef struct {
typedef struct
{
pio_file_kind_e kind;
mode_t mode;
size_t size;
time_t mtime;
@@ -258,6 +259,110 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size)
IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size);
}
pio_file_kind_e
pio_statmode2file_kind(mode_t mode, const char* path)
{
pio_file_kind_e kind;
if (S_ISREG(mode))
kind = PIO_KIND_REGULAR;
else if (S_ISDIR(mode))
kind = PIO_KIND_DIRECTORY;
#ifdef S_ISLNK
else if (S_ISLNK(mode))
kind = PIO_KIND_SYMLINK;
#endif
#ifdef S_ISFIFO
else if (S_ISFIFO(mode))
kind = PIO_KIND_FIFO;
#endif
#ifdef S_ISSOCK
else if (S_ISFIFO(mode))
kind = PIO_KIND_SOCK;
#endif
#ifdef S_ISCHR
else if (S_ISCHR(mode))
kind = PIO_KIND_CHARDEV;
#endif
#ifdef S_ISBLK
else if (S_ISBLK(mode))
kind = PIO_KIND_BLOCKDEV;
#endif
else
elog(ERROR, "Unsupported file mode kind \"%x\" for file '%s'",
mode, path);
return kind;
}
pio_file_kind_e
pio_str2file_kind(const char* str, const char* path)
{
pio_file_kind_e kind;
if (strncmp(str, "reg", 3) == 0)
kind = PIO_KIND_REGULAR;
else if (strncmp(str, "dir", 3) == 0)
kind = PIO_KIND_DIRECTORY;
else if (strncmp(str, "sym", 3) == 0)
kind = PIO_KIND_SYMLINK;
else if (strncmp(str, "fifo", 4) == 0)
kind = PIO_KIND_FIFO;
else if (strncmp(str, "sock", 4) == 0)
kind = PIO_KIND_SOCK;
else if (strncmp(str, "chdev", 5) == 0)
kind = PIO_KIND_CHARDEV;
else if (strncmp(str, "bldev", 5) == 0)
kind = PIO_KIND_BLOCKDEV;
else
elog(ERROR, "Unknown file kind \"%s\" for file '%s'",
str, path);
return kind;
}
const char*
pio_file_kind2str(pio_file_kind_e kind, const char *path)
{
switch (kind)
{
case PIO_KIND_REGULAR:
return "reg";
case PIO_KIND_DIRECTORY:
return "dir";
case PIO_KIND_SYMLINK:
return "sym";
case PIO_KIND_FIFO:
return "fifo";
case PIO_KIND_SOCK:
return "sock";
case PIO_KIND_CHARDEV:
return "chdev";
case PIO_KIND_BLOCKDEV:
return "bldev";
default:
elog(ERROR, "Unknown file kind \"%d\" for file '%s'",
kind, path);
}
return NULL;
}
#ifndef S_ISGID
#define S_ISGID 0
#endif
#ifndef S_ISUID
#define S_ISUID 0
#endif
#ifndef S_ISVTX
#define S_ISVTX 0
#endif
mode_t
pio_limit_mode(mode_t mode)
{
if (S_ISDIR(mode))
mode &= 0x1ff | S_ISGID | S_ISUID | S_ISVTX;
else
mode &= 0x1ff;
return mode;
}
/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */
FILE*
fio_open_stream(fio_location location, const char* path)
@@ -1094,66 +1199,6 @@ fio_read(int fd, void* buf, size_t size)
}
}
/* Get information about file */
int
fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlink)
{
if (fio_is_remote(location))
{
fio_header hdr = {
.cop = FIO_STAT,
.handle = -1,
.size = strlen(path) + 1,
.arg = follow_symlink,
};
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size);
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
Assert(hdr.cop == FIO_STAT);
IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st));
if (hdr.arg != 0)
{
errno = hdr.arg;
return -1;
}
return 0;
}
else
{
return follow_symlink ? stat(path, st) : lstat(path, st);
}
}
/*
* Compare, that filename1 and filename2 is the same file
* in windows compare only filenames
*/
bool
fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink)
{
#ifndef WIN32
struct stat stat1, stat2;
if (fio_stat(location, filename1, &stat1, follow_symlink) < 0)
elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno));
if (fio_stat(location, filename2, &stat2, follow_symlink) < 0)
elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno));
return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev;
#else
char *abs_name1 = make_absolute_path(filename1);
char *abs_name2 = make_absolute_path(filename2);
bool result = strcmp(abs_name1, abs_name2) == 0;
free(abs_name2);
free(abs_name1);
return result;
#endif
}
/*
* Read value of a symbolic link
* this is a wrapper about readlink() syscall
@@ -3263,6 +3308,7 @@ fio_list_dir_impl(int out, char* buf)
fio_pgFile fio_file;
pgFile *file = (pgFile *) parray_get(file_files, i);
fio_file.kind = file->kind;
fio_file.mode = file->mode;
fio_file.size = file->size;
fio_file.mtime = file->mtime;
@@ -3552,10 +3598,16 @@ fio_communicate(int in, int out)
size_t buf_size = 128*1024;
char* buf = (char*)pgut_malloc(buf_size);
fio_header hdr;
struct stat st;
pioDrive_i drive;
pio_stat_t st;
int rc;
int tmp_fd;
pg_crc32 crc;
err_i err = $noerr();
FOBJ_FUNC_ARP();
drive = pioDriveForLocation(FIO_LOCAL_HOST);
#ifdef WIN32
SYS_CHECK(setmode(in, _O_BINARY));
@@ -3564,6 +3616,7 @@ fio_communicate(int in, int out)
/* Main loop until end of processing all master commands */
while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) {
FOBJ_LOOP_ARP();
if (hdr.size != 0) {
if (hdr.size > buf_size) {
/* Extend buffer on demand */
@@ -3656,11 +3709,17 @@ fio_communicate(int in, int out)
}
case FIO_STAT: /* Get information about file with specified path */
hdr.size = sizeof(st);
rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st);
hdr.arg = rc < 0 ? errno : 0;
st = $i(pioStat, drive, buf, .follow_symlink = hdr.arg != 0,
.err = &err);
hdr.arg = $haserr(err) ? getErrno(err) : 0;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st));
break;
case FIO_FILES_ARE_SAME:
hdr.arg = (int)$i(pioFilesAreSame, drive, buf, buf+strlen(buf)+1);
hdr.size = 0;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
break;
case FIO_ACCESS: /* Check presence of file with specified name */
hdr.size = 0;
hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0;
@@ -3903,7 +3962,7 @@ pioFile_fobjDispose(VSelf)
static bool
common_pioExists(fobj_t self, path_t path, err_i *err)
{
struct stat buf;
pio_stat_t buf;
fobj_reset_err(err);
/* follow symlink ? */
@@ -3913,7 +3972,7 @@ common_pioExists(fobj_t self, path_t path, err_i *err)
*err = $noerr();
return false;
}
if ($noerr(*err) && !S_ISREG(buf.st_mode))
if ($noerr(*err) && buf.pst_kind != PIO_KIND_REGULAR)
*err = $err(SysErr, "File {path:q} is not regular", path(path));
if ($haserr(*err)) {
*err = $syserr(getErrno(*err), "Could not check file existance: {cause:$M}",
@@ -3947,17 +4006,52 @@ pioLocalDrive_pioOpen(VSelf, path_t path, int flags,
return bind_pioFile(file);
}
static struct stat
static pio_stat_t
pioLocalDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
{
struct stat st = {0};
pio_stat_t pst = {0};
int r;
fobj_reset_err(err);
r = follow_symlink ? stat(path, &st) : lstat(path, &st);
if (r < 0)
*err = $syserr(errno, "Cannot stat file {path:q}", path(path));
return st;
else
{
pst.pst_kind = pio_statmode2file_kind(st.st_mode, path);
pst.pst_mode = pio_limit_mode(st.st_mode);
pst.pst_size = st.st_size;
pst.pst_mtime = st.st_mtime;
}
return pst;
}
/*
* Compare, that filename1 and filename2 is the same file
* in windows compare only filenames
*/
static bool
pioLocalDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2)
{
#ifndef WIN32
struct stat stat1, stat2;
if (stat(file1, &stat1) < 0)
elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno));
if (stat(file2, &stat2) < 0)
elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno));
return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev;
#else
char *abs_name1 = make_absolute_path(file1);
char *abs_name2 = make_absolute_path(file2);
bool result = strcmp(abs_name1, abs_name2) == 0;
free(abs_name2);
free(abs_name1);
return result;
#endif
}
#define pioLocalDrive_pioExists common_pioExists
@@ -4213,10 +4307,10 @@ pioRemoteDrive_pioOpen(VSelf, path_t path,
return bind_pioFile(file);
}
static struct stat
static pio_stat_t
pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
{
struct stat st = {0};
pio_stat_t st = {0};
fio_header hdr = {
.cop = FIO_STAT,
.handle = -1,
@@ -4240,6 +4334,32 @@ pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
return st;
}
static bool
pioRemoteDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2)
{
fio_header hdr = {
.cop = FIO_FILES_ARE_SAME,
.handle = -1,
.arg = 0,
};
char _buf[512];
ft_strbuf_t buf = ft_strbuf_init_stack(_buf, sizeof(_buf));
ft_strbuf_catc(&buf, file1);
ft_strbuf_cat1(&buf, '\x00');
ft_strbuf_catc(&buf, file2);
hdr.size = buf.len + 1;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len+1), buf.len+1);
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
ft_dbg_assert(hdr.cop == FIO_FILES_ARE_SAME);
ft_strbuf_free(&buf);
return hdr.arg == 1;
}
#define pioRemoteDrive_pioExists common_pioExists
static err_i
@@ -4388,6 +4508,7 @@ pioRemoteDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_ta
/* receive metainformation */
IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file));
file->kind = fio_file.kind;
file->mode = fio_file.mode;
file->size = fio_file.size;
file->mtime = fio_file.mtime;
+30 -9
View File
@@ -67,7 +67,8 @@ typedef enum
FIO_READLINK,
FIO_SYNC_FILE,
FIO_SEND_FILE_CONTENT,
FIO_PAGE_ZERO
FIO_PAGE_ZERO,
FIO_FILES_ARE_SAME,
} fio_operations;
typedef struct
@@ -92,6 +93,24 @@ typedef enum
FIO_REMOTE_HOST /* date is located at remote host */
} fio_location;
typedef enum pio_file_kind {
PIO_KIND_UNKNOWN = 0,
PIO_KIND_REGULAR = 1,
PIO_KIND_DIRECTORY = 2,
PIO_KIND_SYMLINK = 3,
PIO_KIND_FIFO = 4,
PIO_KIND_SOCK = 5,
PIO_KIND_CHARDEV = 6,
PIO_KIND_BLOCKDEV = 7,
} pio_file_kind_e;
typedef struct pio_stat {
uint64_t pst_size;
int64_t pst_mtime;
uint32_t pst_mode;
pio_file_kind_e pst_kind;
} pio_stat_t;
extern fio_location MyLocation;
extern void setMyLocation(ProbackupSubcmd const subcmd);
@@ -125,7 +144,6 @@ extern int fio_check_error_fd_gz(gzFile f, char **errmsg);
extern ssize_t fio_read(int fd, void* buf, size_t size);
extern int fio_flush(int fd);
extern int fio_seek(int fd, off_t offs);
extern int fio_fstat(int fd, struct stat* st);
extern int fio_truncate(int fd, off_t size);
extern int fio_close(int fd);
@@ -142,7 +160,6 @@ extern int fio_fflush(FILE* f);
extern int fio_fseek(FILE* f, off_t offs);
extern int fio_ftruncate(FILE* f, off_t size);
extern int fio_fclose(FILE* f);
extern int fio_ffstat(FILE* f, struct stat* st);
extern FILE* fio_open_stream(fio_location location, const char* name);
extern int fio_close_stream(FILE* f);
@@ -175,8 +192,6 @@ extern int fio_symlink(fio_location location, const char* target, const char
extern int fio_remove(fio_location location, const char* path, bool missing_ok);
extern int fio_chmod(fio_location location, const char* path, int mode);
extern int fio_access(fio_location location, const char* path, int mode);
extern int fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlinks);
extern bool fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink);
extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz);
extern pid_t fio_check_postmaster(fio_location location, const char *pgdata);
@@ -198,6 +213,10 @@ extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok);
#endif
extern pg_crc32 pgFileGetCRC32Cgz(const char *file_path, bool missing_ok);
extern pio_file_kind_e pio_statmode2file_kind(mode_t mode, const char* path);
extern pio_file_kind_e pio_str2file_kind(const char* str, const char* path);
extern const char* pio_file_kind2str(pio_file_kind_e kind, const char* path);
extern mode_t pio_limit_mode(mode_t mode);
// OBJECTS
@@ -237,19 +256,19 @@ fobj_iface(pioWriteFlush);
fobj_iface(pioWriteCloser);
fobj_iface(pioReadCloser);
typedef struct stat stat_t;
// Drive
#define mth__pioOpen pioFile_i, (path_t, path), (int, flags), \
(int, permissions), (err_i *, err)
#define mth__pioOpen__optional() (permissions, FILE_PERMISSION)
#define mth__pioStat stat_t, (path_t, path), (bool, follow_symlink), \
#define mth__pioStat pio_stat_t, (path_t, path), (bool, follow_symlink), \
(err_i *, err)
#define mth__pioRemove err_i, (path_t, path), (bool, missing_ok)
#define mth__pioRename err_i, (path_t, old_path), (path_t, new_path)
#define mth__pioExists bool, (path_t, path), (err_i *, err)
#define mth__pioGetCRC32 pg_crc32, (path_t, path), (bool, compressed), \
(err_i *, err)
/* Compare, that filename1 and filename2 is the same file */
#define mth__pioFilesAreSame bool, (path_t, file1), (path_t, file2)
#define mth__pioIsRemote bool
#define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict)
#define mth__pioListDir void, (parray *, files), (const char *, root), \
@@ -265,12 +284,14 @@ fobj_method(pioExists);
fobj_method(pioIsRemote);
fobj_method(pioGetCRC32);
fobj_method(pioMakeDir);
fobj_method(pioFilesAreSame);
fobj_method(pioListDir);
fobj_method(pioRemoveDir);
#define iface__pioDrive mth(pioOpen, pioStat, pioRemove, pioRename), \
mth(pioExists, pioGetCRC32, pioIsRemote), \
mth(pioMakeDir, pioListDir, pioRemoveDir)
mth(pioMakeDir, pioListDir, pioRemoveDir), \
mth(pioFilesAreSame)
fobj_iface(pioDrive);
extern pioDrive_i pioDriveForLocation(fio_location location);
+8 -5
View File
@@ -55,6 +55,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
pthread_t *threads;
validate_files_arg *threads_args;
int i;
err_i err;
// parray *dbOid_exclude_list = NULL;
/* Check backup program version */
@@ -199,14 +200,16 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
(parse_program_version(backup->program_version) == 20201)))
{
char path[MAXPGPATH];
struct stat st;
pio_stat_t st;
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
if (fio_stat(FIO_BACKUP_HOST, path, &st, true) < 0)
elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno));
st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST),
.path = path, .follow_symlink = true, .err = &err);
if ($haserr(err))
ft_logerr(FT_FATAL, $errmsg(err), "");
if (st.st_size >= (BLCKSZ*500))
if (st.pst_size >= (BLCKSZ*500))
{
elog(WARNING, "Backup %s is a victim of metadata corruption. "
"Additional information can be found here: "
@@ -242,7 +245,7 @@ pgBackupValidateFiles(void *arg)
elog(ERROR, "Interrupted during validate");
/* Validate only regular files */
if (!S_ISREG(file->mode))
if (file->kind != PIO_KIND_REGULAR)
continue;
/*