You've already forked pg_probackup
mirror of
https://github.com/postgrespro/pg_probackup.git
synced 2026-06-21 01:34:15 +02:00
Merge branch 'REL_2_6_rework_stat' into REL_2_6
This commit is contained in:
Yura Sokolov
+4
-4
@@ -388,9 +388,9 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
|
||||
char to_fullpath[MAXPGPATH];
|
||||
char to_fullpath_part[MAXPGPATH];
|
||||
/* partial handling */
|
||||
struct stat st;
|
||||
pio_stat_t st;
|
||||
int partial_try_count = 0;
|
||||
ssize_t partial_file_size = 0;
|
||||
int64_t partial_file_size = 0;
|
||||
bool partial_is_stale = true;
|
||||
size_t len;
|
||||
err_i err = $noerr();
|
||||
@@ -466,11 +466,11 @@ push_file_internal(const char *wal_file_name, const char *pg_xlog_dir,
|
||||
elog(LOG,
|
||||
"Temp WAL file already exists, waiting on it %u seconds: \"%s\"",
|
||||
archive_timeout, to_fullpath_part);
|
||||
partial_file_size = st.st_size;
|
||||
partial_file_size = st.pst_size;
|
||||
}
|
||||
|
||||
/* file size is changing */
|
||||
if (st.st_size != partial_file_size)
|
||||
if (st.pst_size != partial_file_size)
|
||||
partial_is_stale = false;
|
||||
|
||||
sleep(1);
|
||||
|
||||
+11
-11
@@ -391,7 +391,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
|
||||
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
|
||||
|
||||
/* if the entry was a directory, create it in the backup */
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
{
|
||||
char dirpath[MAXPGPATH];
|
||||
|
||||
@@ -568,7 +568,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
|
||||
pgFile *file = (pgFile *) parray_get(backup_files_list, i);
|
||||
|
||||
/* TODO: sync directory ? */
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
if (file->write_size <= 0)
|
||||
@@ -1836,7 +1836,7 @@ pg_stop_backup_write_file_helper(const char *path, const char *filename, const c
|
||||
file = pgFileNew(full_filename, filename, true, 0,
|
||||
FIO_BACKUP_HOST);
|
||||
|
||||
if (S_ISREG(file->mode))
|
||||
if (file->kind == PIO_KIND_REGULAR)
|
||||
{
|
||||
file->crc = pgFileGetCRC32C(full_filename, false);
|
||||
|
||||
@@ -1990,7 +1990,7 @@ backup_files(void *arg)
|
||||
pgFile *prev_file = NULL;
|
||||
|
||||
/* We have already copied all directories */
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
if (arguments->thread_num == 1)
|
||||
@@ -2045,9 +2045,9 @@ backup_files(void *arg)
|
||||
}
|
||||
|
||||
/* Encountered some strange beast */
|
||||
if (!S_ISREG(file->mode))
|
||||
elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
|
||||
file->mode, from_fullpath);
|
||||
if (file->kind != PIO_KIND_REGULAR)
|
||||
elog(WARNING, "Unexpected type %s of file \"%s\", skipping",
|
||||
pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
|
||||
|
||||
/* Check that file exist in previous backup */
|
||||
if (current.backup_mode != BACKUP_MODE_FULL)
|
||||
@@ -2120,7 +2120,7 @@ parse_filelist_filenames(parray *files, const char *root)
|
||||
pgFile *file = (pgFile *) parray_get(files, i);
|
||||
int sscanf_result;
|
||||
|
||||
if (S_ISREG(file->mode) &&
|
||||
if (file->kind == PIO_KIND_REGULAR &&
|
||||
path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
|
||||
{
|
||||
/*
|
||||
@@ -2147,7 +2147,7 @@ parse_filelist_filenames(parray *files, const char *root)
|
||||
}
|
||||
}
|
||||
|
||||
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
|
||||
if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 &&
|
||||
file->name && file->name[0])
|
||||
{
|
||||
if (file->forkName == init)
|
||||
@@ -2217,7 +2217,7 @@ set_cfs_datafiles(parray *files, const char *root, char *relative, size_t i)
|
||||
|
||||
if (strstr(prev_file->rel_path, cfs_tblspc_path) != NULL)
|
||||
{
|
||||
if (S_ISREG(prev_file->mode) && prev_file->is_datafile)
|
||||
if (prev_file->kind == PIO_KIND_REGULAR && prev_file->is_datafile)
|
||||
{
|
||||
elog(LOG, "Setting 'is_cfs' on file %s, name %s",
|
||||
prev_file->rel_path, prev_file->name);
|
||||
@@ -2374,7 +2374,7 @@ calculate_datasize_of_filelist(parray *filelist)
|
||||
if (file->external_dir_num != 0 || file->excluded)
|
||||
continue;
|
||||
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
{
|
||||
// TODO is a dir always 4K?
|
||||
bytes += 4096;
|
||||
|
||||
+25
-7
@@ -847,12 +847,22 @@ pgBackupGetBackupMode(pgBackup *backup, bool show_color)
|
||||
static bool
|
||||
IsDir(const char *dirpath, const char *entry, fio_location location)
|
||||
{
|
||||
FOBJ_FUNC_ARP();
|
||||
char path[MAXPGPATH];
|
||||
struct stat st;
|
||||
pio_stat_t st;
|
||||
err_i err;
|
||||
|
||||
join_path_components(path, dirpath, entry);
|
||||
|
||||
return fio_stat(location, path, &st, false) == 0 && S_ISDIR(st.st_mode);
|
||||
st = $i(pioStat, pioDriveForLocation(location),
|
||||
.path = path, .follow_symlink = false, .err = &err);
|
||||
if ($haserr(err))
|
||||
{
|
||||
ft_logerr(FT_WARNING, $errmsg(err), "IsDir");
|
||||
return false;
|
||||
}
|
||||
|
||||
return st.pst_kind == PIO_KIND_DIRECTORY;
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1074,6 +1084,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
|
||||
char path[MAXPGPATH];
|
||||
char linked[MAXPGPATH];
|
||||
char compress_alg_string[MAXPGPATH];
|
||||
char kind[16];
|
||||
int64 write_size,
|
||||
uncompressed_size,
|
||||
mode, /* bit length of mode_t depends on platforms */
|
||||
@@ -1115,6 +1126,11 @@ get_backup_filelist(pgBackup *backup, bool strict)
|
||||
/*
|
||||
* Optional fields
|
||||
*/
|
||||
if (get_control_value_str(buf, "kind", kind, sizeof(kind), false))
|
||||
file->kind = pio_str2file_kind(kind, path);
|
||||
else /* fallback to mode for old backups */
|
||||
file->kind = pio_statmode2file_kind(file->mode, path);
|
||||
|
||||
if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0])
|
||||
{
|
||||
file->linked = pgut_strdup(linked);
|
||||
@@ -1146,7 +1162,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
|
||||
if (!file->is_datafile || file->is_cfs)
|
||||
file->size = file->uncompressed_size;
|
||||
|
||||
if (file->external_dir_num == 0 && S_ISREG(file->mode))
|
||||
if (file->external_dir_num == 0 && file->kind == PIO_KIND_REGULAR)
|
||||
{
|
||||
bool is_datafile = file->is_datafile;
|
||||
set_forkname(file);
|
||||
@@ -2564,14 +2580,14 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
if (file->write_size == FILE_NOT_FOUND)
|
||||
continue;
|
||||
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
{
|
||||
backup_size_on_disk += 4096;
|
||||
uncompressed_size_on_disk += 4096;
|
||||
}
|
||||
|
||||
/* Count the amount of the data actually copied */
|
||||
if (S_ISREG(file->mode) && file->write_size > 0)
|
||||
if (file->kind == PIO_KIND_REGULAR && file->write_size > 0)
|
||||
{
|
||||
/*
|
||||
* Size of WAL files in 'pg_wal' is counted separately
|
||||
@@ -2587,11 +2603,13 @@ write_backup_filelist(pgBackup *backup, parray *files, const char *root,
|
||||
}
|
||||
|
||||
len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", "
|
||||
"\"mode\":\"%u\", \"is_datafile\":\"%u\", "
|
||||
"\"kind\":\"%s\", \"mode\":\"%u\", \"is_datafile\":\"%u\", "
|
||||
"\"is_cfs\":\"%u\", \"crc\":\"%u\", "
|
||||
"\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", "
|
||||
"\"dbOid\":\"%u\"",
|
||||
file->rel_path, file->write_size, file->mode,
|
||||
file->rel_path, file->write_size,
|
||||
pio_file_kind2str(file->kind, file->rel_path),
|
||||
file->mode,
|
||||
file->is_datafile ? 1 : 0,
|
||||
file->is_cfs ? 1 : 0,
|
||||
file->crc,
|
||||
|
||||
+9
-9
@@ -378,7 +378,7 @@ catchup_thread_runner(void *arg)
|
||||
pgFile *dest_file = NULL;
|
||||
|
||||
/* We have already copied all directories */
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
if (file->excluded)
|
||||
@@ -400,9 +400,9 @@ catchup_thread_runner(void *arg)
|
||||
join_path_components(to_fullpath, arguments->to_root, file->rel_path);
|
||||
|
||||
/* Encountered some strange beast */
|
||||
if (!S_ISREG(file->mode))
|
||||
elog(WARNING, "Unexpected type %d of file \"%s\", skipping",
|
||||
file->mode, from_fullpath);
|
||||
if (file->kind != PIO_KIND_REGULAR)
|
||||
elog(WARNING, "Unexpected kind %s of file \"%s\", skipping",
|
||||
pio_file_kind2str(file->kind, from_fullpath), from_fullpath);
|
||||
|
||||
/* Check that file exist in dest pgdata */
|
||||
if (arguments->backup_mode != BACKUP_MODE_FULL)
|
||||
@@ -439,7 +439,7 @@ catchup_thread_runner(void *arg)
|
||||
|
||||
if (file->write_size == BYTES_INVALID)
|
||||
{
|
||||
elog(LOG, "Skipping the unchanged file: \"%s\", read %zu bytes", from_fullpath, file->read_size);
|
||||
elog(LOG, "Skipping the unchanged file: \"%s\", read %lld bytes", from_fullpath, (long long)file->read_size);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -546,7 +546,7 @@ catchup_sync_destination_files(const char* pgdata_path, fio_location location, p
|
||||
* - but PG itself is not relying on fs, its durable_sync
|
||||
* includes directory sync
|
||||
*/
|
||||
if (S_ISDIR(file->mode) || file->excluded)
|
||||
if (file->kind == PIO_KIND_DIRECTORY || file->excluded)
|
||||
continue;
|
||||
|
||||
Assert(file->external_dir_num == 0);
|
||||
@@ -630,8 +630,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
|
||||
|
||||
/* for fancy reporting */
|
||||
time_t start_time, end_time;
|
||||
ssize_t transfered_datafiles_bytes = 0;
|
||||
ssize_t transfered_walfiles_bytes = 0;
|
||||
int64_t transfered_datafiles_bytes = 0;
|
||||
int64_t transfered_walfiles_bytes = 0;
|
||||
char pretty_source_bytes[20];
|
||||
err_i err = $noerr();
|
||||
|
||||
@@ -807,7 +807,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
|
||||
pgFile *file = (pgFile *) parray_get(source_filelist, i);
|
||||
char parent_dir[MAXPGPATH];
|
||||
|
||||
if (!S_ISDIR(file->mode) || file->excluded)
|
||||
if (file->kind != PIO_KIND_DIRECTORY || file->excluded)
|
||||
continue;
|
||||
|
||||
/*
|
||||
|
||||
+2
-2
@@ -147,7 +147,7 @@ check_files(void *arg)
|
||||
elog(ERROR, "interrupted during checkdb");
|
||||
|
||||
/* No need to check directories */
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
if (!pg_atomic_test_set_flag(&file->lock))
|
||||
@@ -161,7 +161,7 @@ check_files(void *arg)
|
||||
elog(INFO, "Progress: (%d/%d). Process file \"%s\"",
|
||||
i + 1, n_files_list, from_fullpath);
|
||||
|
||||
if (S_ISREG(file->mode))
|
||||
if (file->kind == PIO_KIND_REGULAR)
|
||||
{
|
||||
/* check only uncompressed by cfs datafiles */
|
||||
if (file->is_datafile && !file->is_cfs)
|
||||
|
||||
+12
-7
@@ -507,7 +507,8 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
|
||||
* NOTE This is a normal situation, if the file size has changed
|
||||
* since the moment we computed it.
|
||||
*/
|
||||
file->n_blocks = file->size/BLCKSZ;
|
||||
file->n_blocks = (typeof(file->n_blocks))(file->size/BLCKSZ);
|
||||
Assert((int64_t)file->n_blocks * BLCKSZ == file->size);
|
||||
|
||||
/*
|
||||
* Skip unchanged file only if it exists in previous backup.
|
||||
@@ -611,12 +612,15 @@ backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpat
|
||||
elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
|
||||
}
|
||||
|
||||
file->read_size = rc * BLCKSZ;
|
||||
file->read_size = (int64_t)rc * BLCKSZ;
|
||||
|
||||
/* refresh n_blocks for FULL and DELTA */
|
||||
if (backup_mode == BACKUP_MODE_FULL ||
|
||||
backup_mode == BACKUP_MODE_DIFF_DELTA)
|
||||
file->n_blocks = file->read_size / BLCKSZ;
|
||||
{
|
||||
file->n_blocks = (typeof(file->n_blocks))(file->read_size / BLCKSZ);
|
||||
Assert((int64_t)file->n_blocks * BLCKSZ == file->read_size);
|
||||
}
|
||||
|
||||
/* Determine that file didn`t changed in case of incremental backup */
|
||||
if (backup_mode != BACKUP_MODE_FULL &&
|
||||
@@ -650,7 +654,7 @@ cleanup:
|
||||
void
|
||||
catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr sync_lsn, BackupMode backup_mode,
|
||||
uint32 checksum_version, size_t prev_size)
|
||||
uint32 checksum_version, int64_t prev_size)
|
||||
{
|
||||
int rc;
|
||||
bool use_pagemap;
|
||||
@@ -760,7 +764,7 @@ catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpa
|
||||
elog(ERROR, "Cannot read file \"%s\"", from_fullpath);
|
||||
}
|
||||
|
||||
file->read_size = rc * BLCKSZ;
|
||||
file->read_size = (int64_t)rc * BLCKSZ;
|
||||
|
||||
/* Determine that file didn`t changed in case of incremental catchup */
|
||||
if (backup_mode != BACKUP_MODE_FULL &&
|
||||
@@ -1595,7 +1599,8 @@ check_data_file(ConnectionArgs *arguments, pgFile *file,
|
||||
* NOTE This is a normal situation, if the file size has changed
|
||||
* since the moment we computed it.
|
||||
*/
|
||||
nblocks = file->size/BLCKSZ;
|
||||
nblocks = (typeof(nblocks))(file->size/BLCKSZ);
|
||||
Assert((int64_t)nblocks * BLCKSZ == file->size);
|
||||
|
||||
for (blknum = 0; blknum < nblocks; blknum++)
|
||||
{
|
||||
@@ -2275,7 +2280,7 @@ copy_pages(const char *to_fullpath, const char *from_fullpath,
|
||||
elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s",
|
||||
to_fullpath, strerror(errno));
|
||||
{
|
||||
long pos = ftell(out);
|
||||
int64_t pos = ftell(out);
|
||||
|
||||
if (pos < 0)
|
||||
elog(ERROR, "Cannot get position in destination file \"%s\": %s",
|
||||
|
||||
+3
-3
@@ -794,8 +794,8 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
|
||||
char first_to_del_str[MAXFNAMELEN];
|
||||
char oldest_to_keep_str[MAXFNAMELEN];
|
||||
int i;
|
||||
size_t wal_size_logical = 0;
|
||||
size_t wal_size_actual = 0;
|
||||
int64_t wal_size_logical = 0;
|
||||
int64_t wal_size_actual = 0;
|
||||
char wal_pretty_size[20];
|
||||
bool purge_all = false;
|
||||
|
||||
@@ -837,7 +837,7 @@ delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timeli
|
||||
/* sanity */
|
||||
if (OldestToKeepSegNo > FirstToDeleteSegNo)
|
||||
{
|
||||
wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size;
|
||||
wal_size_logical = (int64_t)(OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size;
|
||||
|
||||
/* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */
|
||||
if (purge_all)
|
||||
|
||||
@@ -145,23 +145,26 @@ pgFile *
|
||||
pgFileNew(const char *path, const char *rel_path, bool follow_symlink,
|
||||
int external_dir_num, fio_location location)
|
||||
{
|
||||
struct stat st;
|
||||
FOBJ_FUNC_ARP();
|
||||
pio_stat_t st;
|
||||
pgFile *file;
|
||||
err_i err;
|
||||
|
||||
/* stat the file */
|
||||
if (fio_stat(location, path, &st, follow_symlink) < 0)
|
||||
{
|
||||
st = $i(pioStat, pioDriveForLocation(location), .path = path,
|
||||
.follow_symlink = follow_symlink, .err = &err);
|
||||
if ($haserr(err)) {
|
||||
/* file not found is not an error case */
|
||||
if (errno == ENOENT)
|
||||
if (getErrno(err) == ENOENT)
|
||||
return NULL;
|
||||
elog(ERROR, "cannot stat file \"%s\": %s", path,
|
||||
strerror(errno));
|
||||
ft_logerr(FT_FATAL, $errmsg(err), "pgFileNew");
|
||||
}
|
||||
|
||||
file = pgFileInit(rel_path);
|
||||
file->size = st.st_size;
|
||||
file->mode = st.st_mode;
|
||||
file->mtime = st.st_mtime;
|
||||
file->size = st.pst_size;
|
||||
file->kind = st.pst_kind;
|
||||
file->mode = st.pst_mode;
|
||||
file->mtime = st.pst_mtime;
|
||||
file->external_dir_num = external_dir_num;
|
||||
|
||||
return file;
|
||||
@@ -393,7 +396,7 @@ dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool fol
|
||||
return;
|
||||
}
|
||||
|
||||
if (!S_ISDIR(file->mode))
|
||||
if (file->kind != PIO_KIND_DIRECTORY)
|
||||
{
|
||||
if (external_dir_num > 0)
|
||||
elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected",
|
||||
@@ -435,7 +438,7 @@ dir_check_file(pgFile *file, bool backup_logs)
|
||||
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path);
|
||||
|
||||
/* Check if we need to exclude file by name */
|
||||
if (S_ISREG(file->mode))
|
||||
if (file->kind == PIO_KIND_REGULAR)
|
||||
{
|
||||
for (i = 0; pgdata_exclude_files[i]; i++)
|
||||
if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0)
|
||||
@@ -449,7 +452,7 @@ dir_check_file(pgFile *file, bool backup_logs)
|
||||
* If the directory name is in the exclude list, do not list the
|
||||
* contents.
|
||||
*/
|
||||
else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0)
|
||||
else if (file->kind == PIO_KIND_DIRECTORY && !in_tablespace && file->external_dir_num == 0)
|
||||
{
|
||||
/*
|
||||
* If the item in the exclude list starts with '/', compare to
|
||||
@@ -478,14 +481,14 @@ dir_check_file(pgFile *file, bool backup_logs)
|
||||
}
|
||||
|
||||
/* Do not backup ptrack_init files */
|
||||
if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0)
|
||||
if (file->kind == PIO_KIND_REGULAR && strcmp(file->name, "ptrack_init") == 0)
|
||||
return CHECK_FALSE;
|
||||
|
||||
/*
|
||||
* Check files located inside database directories including directory
|
||||
* 'global'
|
||||
*/
|
||||
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
|
||||
if (file->kind == PIO_KIND_REGULAR && file->tblspcOid != 0 &&
|
||||
file->name && file->name[0])
|
||||
{
|
||||
if (strcmp(file->name, "pg_internal.init") == 0)
|
||||
@@ -607,7 +610,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
|
||||
struct dirent *dent;
|
||||
bool in_tablespace = false;
|
||||
|
||||
if (!S_ISDIR(parent->mode))
|
||||
if (parent->kind != PIO_KIND_DIRECTORY)
|
||||
elog(ERROR, "\"%s\" is not a directory", parent_dir);
|
||||
|
||||
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path);
|
||||
@@ -641,7 +644,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
|
||||
continue;
|
||||
|
||||
/* Skip entries point current dir or parent dir */
|
||||
if (S_ISDIR(file->mode) &&
|
||||
if (file->kind == PIO_KIND_DIRECTORY &&
|
||||
(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
|
||||
{
|
||||
pgFileFree(file);
|
||||
@@ -659,7 +662,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
|
||||
* Add only files, directories and links. Skip sockets and other
|
||||
* unexpected file formats.
|
||||
*/
|
||||
if (!S_ISDIR(file->mode) && !S_ISREG(file->mode))
|
||||
if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR)
|
||||
{
|
||||
elog(WARNING, "Skip '%s': unexpected file format", child);
|
||||
pgFileFree(file);
|
||||
@@ -671,7 +674,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
|
||||
* Do not copy tablespaces twice. It may happen if the tablespace is located
|
||||
* inside the PGDATA.
|
||||
*/
|
||||
if (S_ISDIR(file->mode) &&
|
||||
if (file->kind == PIO_KIND_DIRECTORY &&
|
||||
strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
|
||||
{
|
||||
Oid tblspcOid;
|
||||
@@ -718,7 +721,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
|
||||
* If the entry is a directory call dir_list_file_internal()
|
||||
* recursively.
|
||||
*/
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink,
|
||||
backup_logs, skip_hidden, external_dir_num, location);
|
||||
}
|
||||
@@ -886,7 +889,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
|
||||
{
|
||||
pgFile *file = (pgFile *) parray_get(dest_files, i);
|
||||
|
||||
if (!S_ISDIR(file->mode))
|
||||
if (file->kind != PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
/* skip external directory content */
|
||||
@@ -919,7 +922,7 @@ create_data_directories(parray *dest_files, const char *data_dir, const char *ba
|
||||
char parent_dir[MAXPGPATH];
|
||||
pgFile *dir = (pgFile *) parray_get(dest_files, i);
|
||||
|
||||
if (!S_ISDIR(dir->mode))
|
||||
if (dir->kind != PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
/* skip external directory content */
|
||||
@@ -1523,14 +1526,13 @@ dir_is_empty(const char *path, fio_location location)
|
||||
bool
|
||||
fileExists(const char *path, fio_location location)
|
||||
{
|
||||
struct stat buf;
|
||||
FOBJ_FUNC_ARP();
|
||||
err_i err;
|
||||
bool exists;
|
||||
|
||||
if (fio_stat(location, path, &buf, true) == -1 && errno == ENOENT)
|
||||
return false;
|
||||
else if (!S_ISREG(buf.st_mode))
|
||||
return false;
|
||||
else
|
||||
return true;
|
||||
exists = $i(pioExists, pioDriveForLocation(location), path, &err);
|
||||
|
||||
return exists;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
+12
-6
@@ -28,9 +28,10 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t *
|
||||
{
|
||||
int fd;
|
||||
char *buffer;
|
||||
struct stat statbuf;
|
||||
pio_stat_t statbuf;
|
||||
char fullpath[MAXPGPATH];
|
||||
int len;
|
||||
size_t len;
|
||||
err_i err;
|
||||
|
||||
join_path_components(fullpath, datadir, path);
|
||||
|
||||
@@ -43,16 +44,21 @@ slurpFile(fio_location location, const char *datadir, const char *path, size_t *
|
||||
fullpath, strerror(errno));
|
||||
}
|
||||
|
||||
if (fio_stat(location, fullpath, &statbuf, true) < 0)
|
||||
statbuf = $i(pioStat, pioDriveForLocation(location), .path = fullpath,
|
||||
.follow_symlink = true, .err = &err);
|
||||
if ($haserr(err))
|
||||
{
|
||||
if (safe)
|
||||
return NULL;
|
||||
else
|
||||
elog(ERROR, "Could not stat file \"%s\": %s",
|
||||
fullpath, strerror(errno));
|
||||
ft_logerr(FT_FATAL, $errmsg(err), "slurpFile");
|
||||
}
|
||||
|
||||
len = statbuf.st_size;
|
||||
if (statbuf.pst_size > SSIZE_MAX)
|
||||
ft_log(FT_FATAL, "file \"%s\" is too large: %lld",
|
||||
fullpath, (long long)statbuf.pst_size);
|
||||
|
||||
len = statbuf.pst_size;
|
||||
buffer = pg_malloc(len + 1);
|
||||
|
||||
if (fio_read(fd, buffer, len) != len)
|
||||
|
||||
+3
-2
@@ -639,7 +639,7 @@ merge_chain(InstanceState *instanceState,
|
||||
pgFile *file = (pgFile *) parray_get(dest_backup->files, i);
|
||||
|
||||
/* if the entry was an external directory, create it in the backup */
|
||||
if (file->external_dir_num && S_ISDIR(file->mode))
|
||||
if (file->external_dir_num && file->kind == PIO_KIND_DIRECTORY)
|
||||
{
|
||||
char dirpath[MAXPGPATH];
|
||||
char new_container[MAXPGPATH];
|
||||
@@ -955,6 +955,7 @@ merge_files(void *arg)
|
||||
continue;
|
||||
|
||||
tmp_file = pgFileInit(dest_file->rel_path);
|
||||
tmp_file->kind = dest_file->kind;
|
||||
tmp_file->mode = dest_file->mode;
|
||||
tmp_file->is_datafile = dest_file->is_datafile;
|
||||
tmp_file->is_cfs = dest_file->is_cfs;
|
||||
@@ -962,7 +963,7 @@ merge_files(void *arg)
|
||||
tmp_file->dbOid = dest_file->dbOid;
|
||||
|
||||
/* Directories were created before */
|
||||
if (S_ISDIR(dest_file->mode))
|
||||
if (dest_file->kind == PIO_KIND_DIRECTORY)
|
||||
goto done;
|
||||
|
||||
elog(progress ? INFO : LOG, "Progress: (%d/%zu). Merging file \"%s\"",
|
||||
|
||||
+13
-7
@@ -306,6 +306,7 @@ main(int argc, char *argv[])
|
||||
{
|
||||
char *command = NULL;
|
||||
ProbackupSubcmd backup_subcmd = NO_CMD;
|
||||
err_i err;
|
||||
|
||||
ft_init_log(elog_ft_log);
|
||||
fobj_init();
|
||||
@@ -551,13 +552,17 @@ main(int argc, char *argv[])
|
||||
if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD &&
|
||||
backup_subcmd != ARCHIVE_GET_CMD)
|
||||
{
|
||||
struct stat st;
|
||||
pio_stat_t st;
|
||||
|
||||
if (fio_stat(FIO_BACKUP_HOST, instanceState->instance_backup_subdir_path,
|
||||
&st, true) != 0)
|
||||
st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST),
|
||||
.path = instanceState->instance_backup_subdir_path,
|
||||
.follow_symlink = true,
|
||||
.err = &err);
|
||||
|
||||
if ($haserr(err))
|
||||
{
|
||||
elog(WARNING, "Failed to access directory \"%s\": %s",
|
||||
instanceState->instance_backup_subdir_path, strerror(errno));
|
||||
ft_logerr(FT_WARNING, $errmsg(err), "Failed to access directory \"%s\"",
|
||||
instanceState->instance_backup_subdir_path);
|
||||
|
||||
// TODO: redundant message, should we get rid of it?
|
||||
elog(ERROR, "Instance '%s' does not exist in this backup catalog",
|
||||
@@ -566,7 +571,7 @@ main(int argc, char *argv[])
|
||||
else
|
||||
{
|
||||
/* Ensure that backup_path is a path to a directory */
|
||||
if (!S_ISDIR(st.st_mode))
|
||||
if (st.pst_kind != PIO_KIND_DIRECTORY)
|
||||
elog(ERROR, "-B, --backup-path must be a path to directory");
|
||||
}
|
||||
}
|
||||
@@ -911,7 +916,8 @@ main(int argc, char *argv[])
|
||||
*/
|
||||
char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name);
|
||||
join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR);
|
||||
if (fio_is_same_file(FIO_DB_HOST, stripped_wal_file_path, archive_push_xlog_dir, true))
|
||||
if ($i(pioFilesAreSame, pioDriveForLocation(FIO_DB_HOST),
|
||||
.file1 = stripped_wal_file_path, .file2 = archive_push_xlog_dir))
|
||||
{
|
||||
/* 2nd case */
|
||||
system_id = get_system_identifier(FIO_DB_HOST, instance_config.pgdata, false);
|
||||
|
||||
+12
-10
@@ -215,18 +215,21 @@ typedef enum ForkName
|
||||
typedef struct pgFile
|
||||
{
|
||||
char *name; /* file or directory name */
|
||||
mode_t mode; /* protection (file type and permission) */
|
||||
size_t size; /* size of the file */
|
||||
time_t mtime; /* file st_mtime attribute, can be used only
|
||||
during backup */
|
||||
size_t read_size; /* size of the portion read (if only some pages are
|
||||
|
||||
pio_file_kind_e kind; /* kind of file */
|
||||
uint32_t mode; /* protection (permission) */
|
||||
int64_t size; /* size of the file */
|
||||
|
||||
int64_t read_size; /* size of the portion read (if only some pages are
|
||||
backed up, it's different from size) */
|
||||
int64 write_size; /* size of the backed-up file. BYTES_INVALID means
|
||||
int64_t write_size; /* size of the backed-up file. BYTES_INVALID means
|
||||
that the file existed but was not backed up
|
||||
because not modified since last backup. */
|
||||
size_t uncompressed_size; /* size of the backed-up file before compression
|
||||
int64_t uncompressed_size; /* size of the backed-up file before compression
|
||||
* and adding block headers.
|
||||
*/
|
||||
time_t mtime; /* file st_mtime attribute, can be used only
|
||||
during backup */
|
||||
/* we need int64 here to store '-1' value */
|
||||
pg_crc32 crc; /* CRC value of the file, regular file only */
|
||||
char *rel_path; /* relative path of the file */
|
||||
@@ -594,7 +597,7 @@ struct timelineInfo {
|
||||
XLogSegNo end_segno; /* last present segment in this timeline */
|
||||
size_t n_xlog_files; /* number of segments (only really existing)
|
||||
* does not include lost segments */
|
||||
size_t size; /* space on disk taken by regular WAL files */
|
||||
int64_t size; /* space on disk taken by regular WAL files */
|
||||
parray *backups; /* array of pgBackup sturctures with info
|
||||
* about backups belonging to this timeline */
|
||||
parray *xlog_filelist; /* array of ordinary WAL segments, '.partial'
|
||||
@@ -1040,7 +1043,6 @@ extern bool backup_contains_external(const char *dir, parray *dirs_list);
|
||||
extern bool dir_is_empty(const char *path, fio_location location);
|
||||
|
||||
extern bool fileExists(const char *path, fio_location location);
|
||||
extern size_t pgFileSize(const char *path);
|
||||
|
||||
extern pgFile *pgFileNew(const char *path, const char *rel_path,
|
||||
bool follow_symlink, int external_dir_num,
|
||||
@@ -1071,7 +1073,7 @@ extern bool check_data_file(ConnectionArgs *arguments, pgFile *file,
|
||||
|
||||
extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr sync_lsn, BackupMode backup_mode,
|
||||
uint32 checksum_version, size_t prev_size);
|
||||
uint32 checksum_version, int64_t prev_size);
|
||||
extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath,
|
||||
XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode,
|
||||
CompressAlg calg, int clevel, uint32 checksum_version,
|
||||
|
||||
+13
-13
@@ -837,11 +837,11 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
{
|
||||
pgFile *file = (pgFile *) parray_get(dest_files, i);
|
||||
|
||||
if (S_ISDIR(file->mode))
|
||||
if (file->kind == PIO_KIND_DIRECTORY)
|
||||
total_bytes += 4096;
|
||||
|
||||
if (!params->skip_external_dirs &&
|
||||
file->external_dir_num && S_ISDIR(file->mode))
|
||||
file->external_dir_num && file->kind == PIO_KIND_DIRECTORY)
|
||||
{
|
||||
char *external_path;
|
||||
char dirpath[MAXPGPATH];
|
||||
@@ -926,7 +926,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
redundant = true;
|
||||
|
||||
/* do not delete the useful internal directories */
|
||||
if (S_ISDIR(file->mode) && !redundant)
|
||||
if (file->kind == PIO_KIND_DIRECTORY && !redundant)
|
||||
continue;
|
||||
|
||||
/* if file does not exists in destination list, then we can safely unlink it */
|
||||
@@ -1056,7 +1056,7 @@ restore_chain(pgBackup *dest_backup, parray *parent_chain,
|
||||
char to_fullpath[MAXPGPATH];
|
||||
pgFile *dest_file = (pgFile *) parray_get(dest_files, i);
|
||||
|
||||
if (S_ISDIR(dest_file->mode))
|
||||
if (dest_file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
/* skip external files if ordered to do so */
|
||||
@@ -1137,7 +1137,7 @@ restore_files(void *arg)
|
||||
pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i);
|
||||
|
||||
/* Directories were created before */
|
||||
if (S_ISDIR(dest_file->mode))
|
||||
if (dest_file->kind == PIO_KIND_DIRECTORY)
|
||||
continue;
|
||||
|
||||
if (!pg_atomic_test_set_flag(&dest_file->lock))
|
||||
@@ -1543,13 +1543,14 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup,
|
||||
char path[MAXPGPATH];
|
||||
FILE *fp = NULL;
|
||||
FILE *fp_tmp = NULL;
|
||||
struct stat st;
|
||||
pio_stat_t st;
|
||||
char current_time_str[100];
|
||||
/* postgresql.auto.conf parsing */
|
||||
char line[16384] = "\0";
|
||||
char *buf = NULL;
|
||||
int buf_len = 0;
|
||||
int buf_len_max = 16384;
|
||||
err_i err;
|
||||
|
||||
elog(LOG, "update recovery settings in postgresql.auto.conf");
|
||||
|
||||
@@ -1557,17 +1558,16 @@ update_recovery_options(InstanceState *instanceState, pgBackup *backup,
|
||||
|
||||
join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf");
|
||||
|
||||
if (fio_stat(FIO_DB_HOST, postgres_auto_path, &st, false) < 0)
|
||||
{
|
||||
st = $i(pioStat, pioDriveForLocation(FIO_DB_HOST),
|
||||
.path = postgres_auto_path, .follow_symlink = false, .err = &err);
|
||||
/* file not found is not an error case */
|
||||
if (errno != ENOENT)
|
||||
elog(ERROR, "cannot stat file \"%s\": %s", postgres_auto_path,
|
||||
strerror(errno));
|
||||
st.st_size = 0;
|
||||
if ($haserr(err) && getErrno(err) != ENOENT)
|
||||
{
|
||||
ft_logerr(FT_FATAL, $errmsg(err), "");
|
||||
}
|
||||
|
||||
/* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */
|
||||
if (st.st_size > 0)
|
||||
if (st.pst_size > 0)
|
||||
{
|
||||
fp = fio_open_stream(FIO_DB_HOST, postgres_auto_path);
|
||||
if (fp == NULL)
|
||||
|
||||
+1
-1
@@ -1089,7 +1089,7 @@ show_archive_json(const char *instance_name, uint32 xlog_seg_size,
|
||||
appendPQExpBuffer(buf, "%zu", tlinfo->n_xlog_files);
|
||||
|
||||
json_add_key(buf, "size", json_level);
|
||||
appendPQExpBuffer(buf, "%zu", tlinfo->size);
|
||||
appendPQExpBuffer(buf, "%lld", (long long)tlinfo->size);
|
||||
|
||||
json_add_key(buf, "zratio", json_level);
|
||||
|
||||
|
||||
+3
-3
@@ -403,9 +403,9 @@ copy_pgcontrol_file(fio_location from_location, const char *from_fullpath,
|
||||
digestControlFile(&ControlFile, buffer, size);
|
||||
|
||||
file->crc = ControlFile.crc;
|
||||
file->read_size = size;
|
||||
file->write_size = size;
|
||||
file->uncompressed_size = size;
|
||||
file->read_size = (int64_t)size;
|
||||
file->write_size = (int64_t)size;
|
||||
file->uncompressed_size = (int64_t)size;
|
||||
|
||||
writeControlFile(to_location, to_fullpath, &ControlFile);
|
||||
|
||||
|
||||
+192
-70
@@ -55,8 +55,9 @@ typedef struct {
|
||||
|
||||
typedef struct
|
||||
{
|
||||
pio_file_kind_e kind;
|
||||
mode_t mode;
|
||||
size_t size;
|
||||
int64_t size;
|
||||
time_t mtime;
|
||||
bool is_datafile;
|
||||
Oid tblspcOid;
|
||||
@@ -262,6 +263,110 @@ fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size)
|
||||
IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size);
|
||||
}
|
||||
|
||||
pio_file_kind_e
|
||||
pio_statmode2file_kind(mode_t mode, const char* path)
|
||||
{
|
||||
pio_file_kind_e kind;
|
||||
if (S_ISREG(mode))
|
||||
kind = PIO_KIND_REGULAR;
|
||||
else if (S_ISDIR(mode))
|
||||
kind = PIO_KIND_DIRECTORY;
|
||||
#ifdef S_ISLNK
|
||||
else if (S_ISLNK(mode))
|
||||
kind = PIO_KIND_SYMLINK;
|
||||
#endif
|
||||
#ifdef S_ISFIFO
|
||||
else if (S_ISFIFO(mode))
|
||||
kind = PIO_KIND_FIFO;
|
||||
#endif
|
||||
#ifdef S_ISSOCK
|
||||
else if (S_ISFIFO(mode))
|
||||
kind = PIO_KIND_SOCK;
|
||||
#endif
|
||||
#ifdef S_ISCHR
|
||||
else if (S_ISCHR(mode))
|
||||
kind = PIO_KIND_CHARDEV;
|
||||
#endif
|
||||
#ifdef S_ISBLK
|
||||
else if (S_ISBLK(mode))
|
||||
kind = PIO_KIND_BLOCKDEV;
|
||||
#endif
|
||||
else
|
||||
elog(ERROR, "Unsupported file mode kind \"%x\" for file '%s'",
|
||||
mode, path);
|
||||
return kind;
|
||||
}
|
||||
|
||||
pio_file_kind_e
|
||||
pio_str2file_kind(const char* str, const char* path)
|
||||
{
|
||||
pio_file_kind_e kind;
|
||||
if (strncmp(str, "reg", 3) == 0)
|
||||
kind = PIO_KIND_REGULAR;
|
||||
else if (strncmp(str, "dir", 3) == 0)
|
||||
kind = PIO_KIND_DIRECTORY;
|
||||
else if (strncmp(str, "sym", 3) == 0)
|
||||
kind = PIO_KIND_SYMLINK;
|
||||
else if (strncmp(str, "fifo", 4) == 0)
|
||||
kind = PIO_KIND_FIFO;
|
||||
else if (strncmp(str, "sock", 4) == 0)
|
||||
kind = PIO_KIND_SOCK;
|
||||
else if (strncmp(str, "chdev", 5) == 0)
|
||||
kind = PIO_KIND_CHARDEV;
|
||||
else if (strncmp(str, "bldev", 5) == 0)
|
||||
kind = PIO_KIND_BLOCKDEV;
|
||||
else
|
||||
elog(ERROR, "Unknown file kind \"%s\" for file '%s'",
|
||||
str, path);
|
||||
return kind;
|
||||
}
|
||||
|
||||
const char*
|
||||
pio_file_kind2str(pio_file_kind_e kind, const char *path)
|
||||
{
|
||||
switch (kind)
|
||||
{
|
||||
case PIO_KIND_REGULAR:
|
||||
return "reg";
|
||||
case PIO_KIND_DIRECTORY:
|
||||
return "dir";
|
||||
case PIO_KIND_SYMLINK:
|
||||
return "sym";
|
||||
case PIO_KIND_FIFO:
|
||||
return "fifo";
|
||||
case PIO_KIND_SOCK:
|
||||
return "sock";
|
||||
case PIO_KIND_CHARDEV:
|
||||
return "chdev";
|
||||
case PIO_KIND_BLOCKDEV:
|
||||
return "bldev";
|
||||
default:
|
||||
elog(ERROR, "Unknown file kind \"%d\" for file '%s'",
|
||||
kind, path);
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#ifndef S_ISGID
|
||||
#define S_ISGID 0
|
||||
#endif
|
||||
#ifndef S_ISUID
|
||||
#define S_ISUID 0
|
||||
#endif
|
||||
#ifndef S_ISVTX
|
||||
#define S_ISVTX 0
|
||||
#endif
|
||||
|
||||
mode_t
|
||||
pio_limit_mode(mode_t mode)
|
||||
{
|
||||
if (S_ISDIR(mode))
|
||||
mode &= 0x1ff | S_ISGID | S_ISUID | S_ISVTX;
|
||||
else
|
||||
mode &= 0x1ff;
|
||||
return mode;
|
||||
}
|
||||
|
||||
/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */
|
||||
FILE*
|
||||
fio_open_stream(fio_location location, const char* path)
|
||||
@@ -1098,66 +1203,6 @@ fio_read(int fd, void* buf, size_t size)
|
||||
}
|
||||
}
|
||||
|
||||
/* Get information about file */
|
||||
int
|
||||
fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlink)
|
||||
{
|
||||
if (fio_is_remote(location))
|
||||
{
|
||||
fio_header hdr = {
|
||||
.cop = FIO_STAT,
|
||||
.handle = -1,
|
||||
.size = strlen(path) + 1,
|
||||
.arg = follow_symlink,
|
||||
};
|
||||
|
||||
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size);
|
||||
|
||||
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
Assert(hdr.cop == FIO_STAT);
|
||||
IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st));
|
||||
|
||||
if (hdr.arg != 0)
|
||||
{
|
||||
errno = hdr.arg;
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
return follow_symlink ? stat(path, st) : lstat(path, st);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare, that filename1 and filename2 is the same file
|
||||
* in windows compare only filenames
|
||||
*/
|
||||
bool
|
||||
fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink)
|
||||
{
|
||||
#ifndef WIN32
|
||||
struct stat stat1, stat2;
|
||||
|
||||
if (fio_stat(location, filename1, &stat1, follow_symlink) < 0)
|
||||
elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno));
|
||||
|
||||
if (fio_stat(location, filename2, &stat2, follow_symlink) < 0)
|
||||
elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno));
|
||||
|
||||
return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev;
|
||||
#else
|
||||
char *abs_name1 = make_absolute_path(filename1);
|
||||
char *abs_name2 = make_absolute_path(filename2);
|
||||
bool result = strcmp(abs_name1, abs_name2) == 0;
|
||||
free(abs_name2);
|
||||
free(abs_name1);
|
||||
return result;
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* Read value of a symbolic link
|
||||
* this is a wrapper about readlink() syscall
|
||||
@@ -1873,6 +1918,7 @@ fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file,
|
||||
}
|
||||
|
||||
req.arg.nblocks = file->size/BLCKSZ;
|
||||
Assert((int64_t)req.arg.nblocks * BLCKSZ == file->size);
|
||||
req.arg.segmentno = file->segno * RELSEG_SIZE;
|
||||
req.arg.horizonLsn = horizonLsn;
|
||||
req.arg.checksumVersion = checksum_version;
|
||||
@@ -3267,6 +3313,7 @@ fio_list_dir_impl(int out, char* buf)
|
||||
fio_pgFile fio_file;
|
||||
pgFile *file = (pgFile *) parray_get(file_files, i);
|
||||
|
||||
fio_file.kind = file->kind;
|
||||
fio_file.mode = file->mode;
|
||||
fio_file.size = file->size;
|
||||
fio_file.mtime = file->mtime;
|
||||
@@ -3556,10 +3603,16 @@ fio_communicate(int in, int out)
|
||||
size_t buf_size = 128*1024;
|
||||
char* buf = (char*)pgut_malloc(buf_size);
|
||||
fio_header hdr;
|
||||
struct stat st;
|
||||
pioDrive_i drive;
|
||||
pio_stat_t st;
|
||||
int rc;
|
||||
int tmp_fd;
|
||||
pg_crc32 crc;
|
||||
err_i err = $noerr();
|
||||
|
||||
FOBJ_FUNC_ARP();
|
||||
|
||||
drive = pioDriveForLocation(FIO_LOCAL_HOST);
|
||||
|
||||
#ifdef WIN32
|
||||
SYS_CHECK(setmode(in, _O_BINARY));
|
||||
@@ -3568,6 +3621,7 @@ fio_communicate(int in, int out)
|
||||
|
||||
/* Main loop until end of processing all master commands */
|
||||
while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) {
|
||||
FOBJ_LOOP_ARP();
|
||||
if (hdr.size != 0) {
|
||||
if (hdr.size > buf_size) {
|
||||
/* Extend buffer on demand */
|
||||
@@ -3660,11 +3714,17 @@ fio_communicate(int in, int out)
|
||||
}
|
||||
case FIO_STAT: /* Get information about file with specified path */
|
||||
hdr.size = sizeof(st);
|
||||
rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st);
|
||||
hdr.arg = rc < 0 ? errno : 0;
|
||||
st = $i(pioStat, drive, buf, .follow_symlink = hdr.arg != 0,
|
||||
.err = &err);
|
||||
hdr.arg = $haserr(err) ? getErrno(err) : 0;
|
||||
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st));
|
||||
break;
|
||||
case FIO_FILES_ARE_SAME:
|
||||
hdr.arg = (int)$i(pioFilesAreSame, drive, buf, buf+strlen(buf)+1);
|
||||
hdr.size = 0;
|
||||
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
break;
|
||||
case FIO_ACCESS: /* Check presence of file with specified name */
|
||||
hdr.size = 0;
|
||||
hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0;
|
||||
@@ -3914,7 +3974,7 @@ pioFile_fobjDispose(VSelf)
|
||||
static bool
|
||||
common_pioExists(fobj_t self, path_t path, err_i *err)
|
||||
{
|
||||
struct stat buf;
|
||||
pio_stat_t buf;
|
||||
fobj_reset_err(err);
|
||||
|
||||
/* follow symlink ? */
|
||||
@@ -3924,7 +3984,7 @@ common_pioExists(fobj_t self, path_t path, err_i *err)
|
||||
*err = $noerr();
|
||||
return false;
|
||||
}
|
||||
if ($noerr(*err) && !S_ISREG(buf.st_mode))
|
||||
if ($noerr(*err) && buf.pst_kind != PIO_KIND_REGULAR)
|
||||
*err = $err(SysErr, "File {path:q} is not regular", path(path));
|
||||
if ($haserr(*err)) {
|
||||
*err = $syserr(getErrno(*err), "Could not check file existance: {cause:$M}",
|
||||
@@ -3958,17 +4018,52 @@ pioLocalDrive_pioOpen(VSelf, path_t path, int flags,
|
||||
return bind_pioFile(file);
|
||||
}
|
||||
|
||||
static struct stat
|
||||
static pio_stat_t
|
||||
pioLocalDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
|
||||
{
|
||||
struct stat st = {0};
|
||||
pio_stat_t pst = {0};
|
||||
int r;
|
||||
fobj_reset_err(err);
|
||||
|
||||
r = follow_symlink ? stat(path, &st) : lstat(path, &st);
|
||||
if (r < 0)
|
||||
*err = $syserr(errno, "Cannot stat file {path:q}", path(path));
|
||||
return st;
|
||||
else
|
||||
{
|
||||
pst.pst_kind = pio_statmode2file_kind(st.st_mode, path);
|
||||
pst.pst_mode = pio_limit_mode(st.st_mode);
|
||||
pst.pst_size = st.st_size;
|
||||
pst.pst_mtime = st.st_mtime;
|
||||
}
|
||||
return pst;
|
||||
}
|
||||
|
||||
/*
|
||||
* Compare, that filename1 and filename2 is the same file
|
||||
* in windows compare only filenames
|
||||
*/
|
||||
static bool
|
||||
pioLocalDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2)
|
||||
{
|
||||
#ifndef WIN32
|
||||
struct stat stat1, stat2;
|
||||
|
||||
if (stat(file1, &stat1) < 0)
|
||||
elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno));
|
||||
|
||||
if (stat(file2, &stat2) < 0)
|
||||
elog(ERROR, "Can't stat file \"%s\": %s", file1, strerror(errno));
|
||||
|
||||
return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev;
|
||||
#else
|
||||
char *abs_name1 = make_absolute_path(file1);
|
||||
char *abs_name2 = make_absolute_path(file2);
|
||||
bool result = strcmp(abs_name1, abs_name2) == 0;
|
||||
free(abs_name2);
|
||||
free(abs_name1);
|
||||
return result;
|
||||
#endif
|
||||
}
|
||||
|
||||
#define pioLocalDrive_pioExists common_pioExists
|
||||
@@ -4224,10 +4319,10 @@ pioRemoteDrive_pioOpen(VSelf, path_t path,
|
||||
return bind_pioFile(file);
|
||||
}
|
||||
|
||||
static struct stat
|
||||
static pio_stat_t
|
||||
pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
|
||||
{
|
||||
struct stat st = {0};
|
||||
pio_stat_t st = {0};
|
||||
fio_header hdr = {
|
||||
.cop = FIO_STAT,
|
||||
.handle = -1,
|
||||
@@ -4251,6 +4346,32 @@ pioRemoteDrive_pioStat(VSelf, path_t path, bool follow_symlink, err_i *err)
|
||||
return st;
|
||||
}
|
||||
|
||||
static bool
|
||||
pioRemoteDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2)
|
||||
{
|
||||
fio_header hdr = {
|
||||
.cop = FIO_FILES_ARE_SAME,
|
||||
.handle = -1,
|
||||
.arg = 0,
|
||||
};
|
||||
char _buf[512];
|
||||
ft_strbuf_t buf = ft_strbuf_init_stack(_buf, sizeof(_buf));
|
||||
ft_strbuf_catc(&buf, file1);
|
||||
ft_strbuf_cat1(&buf, '\x00');
|
||||
ft_strbuf_catc(&buf, file2);
|
||||
hdr.size = buf.len + 1;
|
||||
|
||||
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
IO_CHECK(fio_write_all(fio_stdout, buf.ptr, buf.len+1), buf.len+1);
|
||||
|
||||
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
|
||||
ft_dbg_assert(hdr.cop == FIO_FILES_ARE_SAME);
|
||||
|
||||
ft_strbuf_free(&buf);
|
||||
|
||||
return hdr.arg == 1;
|
||||
}
|
||||
|
||||
#define pioRemoteDrive_pioExists common_pioExists
|
||||
|
||||
static err_i
|
||||
@@ -4399,6 +4520,7 @@ pioRemoteDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_ta
|
||||
/* receive metainformation */
|
||||
IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file));
|
||||
|
||||
file->kind = fio_file.kind;
|
||||
file->mode = fio_file.mode;
|
||||
file->size = fio_file.size;
|
||||
file->mtime = fio_file.mtime;
|
||||
|
||||
+30
-9
@@ -67,7 +67,8 @@ typedef enum
|
||||
FIO_READLINK,
|
||||
FIO_SYNC_FILE,
|
||||
FIO_SEND_FILE_CONTENT,
|
||||
FIO_PAGE_ZERO
|
||||
FIO_PAGE_ZERO,
|
||||
FIO_FILES_ARE_SAME,
|
||||
} fio_operations;
|
||||
|
||||
typedef struct
|
||||
@@ -93,6 +94,24 @@ typedef enum
|
||||
FIO_CLOUD_HOST /* date is located at cloud (S3) */
|
||||
} fio_location;
|
||||
|
||||
typedef enum pio_file_kind {
|
||||
PIO_KIND_UNKNOWN = 0,
|
||||
PIO_KIND_REGULAR = 1,
|
||||
PIO_KIND_DIRECTORY = 2,
|
||||
PIO_KIND_SYMLINK = 3,
|
||||
PIO_KIND_FIFO = 4,
|
||||
PIO_KIND_SOCK = 5,
|
||||
PIO_KIND_CHARDEV = 6,
|
||||
PIO_KIND_BLOCKDEV = 7,
|
||||
} pio_file_kind_e;
|
||||
|
||||
typedef struct pio_stat {
|
||||
int64_t pst_size;
|
||||
int64_t pst_mtime;
|
||||
uint32_t pst_mode;
|
||||
pio_file_kind_e pst_kind;
|
||||
} pio_stat_t;
|
||||
|
||||
extern fio_location MyLocation;
|
||||
|
||||
extern void setMyLocation(ProbackupSubcmd const subcmd);
|
||||
@@ -126,7 +145,6 @@ extern int fio_check_error_fd_gz(gzFile f, char **errmsg);
|
||||
extern ssize_t fio_read(int fd, void* buf, size_t size);
|
||||
extern int fio_flush(int fd);
|
||||
extern int fio_seek(int fd, off_t offs);
|
||||
extern int fio_fstat(int fd, struct stat* st);
|
||||
extern int fio_truncate(int fd, off_t size);
|
||||
extern int fio_close(int fd);
|
||||
|
||||
@@ -143,7 +161,6 @@ extern int fio_fflush(FILE* f);
|
||||
extern int fio_fseek(FILE* f, off_t offs);
|
||||
extern int fio_ftruncate(FILE* f, off_t size);
|
||||
extern int fio_fclose(FILE* f);
|
||||
extern int fio_ffstat(FILE* f, struct stat* st);
|
||||
|
||||
extern FILE* fio_open_stream(fio_location location, const char* name);
|
||||
extern int fio_close_stream(FILE* f);
|
||||
@@ -176,8 +193,6 @@ extern int fio_symlink(fio_location location, const char* target, const char
|
||||
extern int fio_remove(fio_location location, const char* path, bool missing_ok);
|
||||
extern int fio_chmod(fio_location location, const char* path, int mode);
|
||||
extern int fio_access(fio_location location, const char* path, int mode);
|
||||
extern int fio_stat(fio_location location, const char* path, struct stat* st, bool follow_symlinks);
|
||||
extern bool fio_is_same_file(fio_location location, const char* filename1, const char* filename2, bool follow_symlink);
|
||||
extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz);
|
||||
extern pid_t fio_check_postmaster(fio_location location, const char *pgdata);
|
||||
|
||||
@@ -199,6 +214,10 @@ extern pg_crc32 pgFileGetCRC32(const char *file_path, bool missing_ok);
|
||||
#endif
|
||||
extern pg_crc32 pgFileGetCRC32Cgz(const char *file_path, bool missing_ok);
|
||||
|
||||
extern pio_file_kind_e pio_statmode2file_kind(mode_t mode, const char* path);
|
||||
extern pio_file_kind_e pio_str2file_kind(const char* str, const char* path);
|
||||
extern const char* pio_file_kind2str(pio_file_kind_e kind, const char* path);
|
||||
extern mode_t pio_limit_mode(mode_t mode);
|
||||
|
||||
// OBJECTS
|
||||
|
||||
@@ -238,19 +257,19 @@ fobj_iface(pioWriteFlush);
|
||||
fobj_iface(pioWriteCloser);
|
||||
fobj_iface(pioReadCloser);
|
||||
|
||||
typedef struct stat stat_t;
|
||||
|
||||
// Drive
|
||||
#define mth__pioOpen pioFile_i, (path_t, path), (int, flags), \
|
||||
(int, permissions), (err_i *, err)
|
||||
#define mth__pioOpen__optional() (permissions, FILE_PERMISSION)
|
||||
#define mth__pioStat stat_t, (path_t, path), (bool, follow_symlink), \
|
||||
#define mth__pioStat pio_stat_t, (path_t, path), (bool, follow_symlink), \
|
||||
(err_i *, err)
|
||||
#define mth__pioRemove err_i, (path_t, path), (bool, missing_ok)
|
||||
#define mth__pioRename err_i, (path_t, old_path), (path_t, new_path)
|
||||
#define mth__pioExists bool, (path_t, path), (err_i *, err)
|
||||
#define mth__pioGetCRC32 pg_crc32, (path_t, path), (bool, compressed), \
|
||||
(err_i *, err)
|
||||
/* Compare, that filename1 and filename2 is the same file */
|
||||
#define mth__pioFilesAreSame bool, (path_t, file1), (path_t, file2)
|
||||
#define mth__pioIsRemote bool
|
||||
#define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict)
|
||||
#define mth__pioListDir void, (parray *, files), (const char *, root), \
|
||||
@@ -266,12 +285,14 @@ fobj_method(pioExists);
|
||||
fobj_method(pioIsRemote);
|
||||
fobj_method(pioGetCRC32);
|
||||
fobj_method(pioMakeDir);
|
||||
fobj_method(pioFilesAreSame);
|
||||
fobj_method(pioListDir);
|
||||
fobj_method(pioRemoveDir);
|
||||
|
||||
#define iface__pioDrive mth(pioOpen, pioStat, pioRemove, pioRename), \
|
||||
mth(pioExists, pioGetCRC32, pioIsRemote), \
|
||||
mth(pioMakeDir, pioListDir, pioRemoveDir)
|
||||
mth(pioMakeDir, pioListDir, pioRemoveDir), \
|
||||
mth(pioFilesAreSame)
|
||||
fobj_iface(pioDrive);
|
||||
|
||||
extern pioDrive_i pioDriveForLocation(fio_location location);
|
||||
|
||||
+8
-5
@@ -55,6 +55,7 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
||||
pthread_t *threads;
|
||||
validate_files_arg *threads_args;
|
||||
int i;
|
||||
err_i err;
|
||||
// parray *dbOid_exclude_list = NULL;
|
||||
|
||||
/* Check backup program version */
|
||||
@@ -199,14 +200,16 @@ pgBackupValidate(pgBackup *backup, pgRestoreParams *params)
|
||||
(parse_program_version(backup->program_version) == 20201)))
|
||||
{
|
||||
char path[MAXPGPATH];
|
||||
struct stat st;
|
||||
pio_stat_t st;
|
||||
|
||||
join_path_components(path, backup->root_dir, DATABASE_FILE_LIST);
|
||||
|
||||
if (fio_stat(FIO_BACKUP_HOST, path, &st, true) < 0)
|
||||
elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno));
|
||||
st = $i(pioStat, pioDriveForLocation(FIO_BACKUP_HOST),
|
||||
.path = path, .follow_symlink = true, .err = &err);
|
||||
if ($haserr(err))
|
||||
ft_logerr(FT_FATAL, $errmsg(err), "");
|
||||
|
||||
if (st.st_size >= (BLCKSZ*500))
|
||||
if (st.pst_size >= (BLCKSZ*500))
|
||||
{
|
||||
elog(WARNING, "Backup %s is a victim of metadata corruption. "
|
||||
"Additional information can be found here: "
|
||||
@@ -242,7 +245,7 @@ pgBackupValidateFiles(void *arg)
|
||||
elog(ERROR, "Interrupted during validate");
|
||||
|
||||
/* Validate only regular files */
|
||||
if (!S_ISREG(file->mode))
|
||||
if (file->kind != PIO_KIND_REGULAR)
|
||||
continue;
|
||||
|
||||
/*
|
||||
|
||||
Reference in New Issue
Block a user