db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Revert "[PBCKP-234] proper pioListDir usage" partially

Browse Source 
This reverts commit 4ecb11f45d.

It seems it would be easier to change to pioOpenDir with rollback.
But we keep pioRemoveDir usage.
This commit is contained in:
Yura Sokolov
2022-12-19 03:48:42 +03:00
parent dfb5b0dc0d
commit 2c756c3397
8 changed files with 212 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backup.c
+3 -5
View File
@@ -275,8 +275,7 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR);
/* list files with the logical path. omit $PGDATA */
db_list_dir(backup_files_list, instance_config.pgdata, true, backup_logs, 0);
exclude_files(backup_files_list, backup_logs);
db_list_dir(backup_files_list, instance_config.pgdata, true, backup_logs, 0, FIO_DB_HOST);
/*
* Get database_map (name to oid) for use in partial restore feature.
@@ -292,9 +291,8 @@ do_backup_pg(InstanceState *instanceState, PGconn *backup_conn,
{
for (i = 0; i < parray_num(external_dirs); i++)
{
/* External dirs numeration starts with 1.
* 0 value is not external dir */
db_list_dir(backup_files_list, parray_get(external_dirs, i), false, false, i+1);
db_list_dir(backup_files_list, parray_get(external_dirs, i),
false, false, i + 1, FIO_DB_HOST);
}
}
src/catchup.c
+2 -4
View File
@@ -648,9 +648,8 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
if (current.backup_mode != BACKUP_MODE_FULL)
{
dest_filelist = parray_new();
db_list_dir(dest_filelist, dest_pgdata, true, false, 0);
db_list_dir(dest_filelist, dest_pgdata, true, false, 0, FIO_LOCAL_HOST);
filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination");
exclude_files(dest_filelist, false);
// fill dest_redo.lsn and dest_redo.tli
get_redo(FIO_LOCAL_HOST, dest_pgdata, &dest_redo);
@@ -720,8 +719,7 @@ do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads,
source_filelist = parray_new();
/* list files with the logical path. omit $PGDATA */
db_list_dir(source_filelist, source_pgdata, true, false, 0);
exclude_files(source_filelist, false);
db_list_dir(source_filelist, source_pgdata, true, false, 0, FIO_DB_HOST);
//REVIEW FIXME. Let's fix that before release.
// TODO what if wal is not a dir (symlink to a dir)?
src/checkdb.c
+1 -2
View File
@@ -206,8 +206,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
files_list = parray_new();
/* list files with the logical path. omit $PGDATA */
db_list_dir(files_list, pgdata, true, false, 0);
exclude_files(files_list, false);
db_list_dir(files_list, pgdata, true, false, 0, FIO_DB_HOST);
/*
* Sort pathname ascending.
src/dir.c
+198 -72
View File
@@ -17,6 +17,10 @@
#include <dirent.h>
#include "utils/configuration.h"
#include "catalog/pg_tablespace.h"
#if PG_VERSION_NUM < 110000
#include "catalog/catalog.h"
#endif
/*
* The contents of these directories are removed or recreated during server
@@ -112,21 +116,17 @@ typedef struct TablespaceCreatedList
TablespaceCreatedListCell *tail;
} TablespaceCreatedList;
typedef struct exclude_cb_ctx {
bool backup_logs;
size_t pref_len;
char exclude_dir_content_pref[MAXPGPATH];
} exclude_cb_ctx;
static char dir_check_file(pgFile *file, bool backup_logs);
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool backup_logs,
int external_dir_num, fio_location location);
static void opt_path_map(ConfigOption *opt, const char *arg,
TablespaceList *list, const char *type);
static void cleanup_tablespace(const char *path);
static void control_string_bad_format(ft_bytes_t str);
static bool exclude_files_cb(void *value, void *exclude_args);
static void print_database_map(ft_strbuf_t *buf, parray *database_list);
@@ -365,6 +365,44 @@ db_map_entry_free(void *entry)
free(entry);
}
/*
* List files and directories in the directory "root" and add
* pgFile objects to "files".
*/
void
db_list_dir(parray *files, const char *root, bool exclude,
bool backup_logs, int external_dir_num,
fio_location location)
{
pgFile *file;
pioDrive_i drive = pioDriveForLocation(location);
file = pgFileNew(root, "", true, external_dir_num, drive);
if (file == NULL)
{
/* For external directory this is not ok */
if (external_dir_num > 0)
elog(ERROR, "External directory is not found: \"%s\"", root);
else
return;
}
if (file->kind != PIO_KIND_DIRECTORY)
{
if (external_dir_num > 0)
elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected",
root);
else
elog(WARNING, "Skip \"%s\": unexpected file format", root);
return;
}
dir_list_file_internal(files, file, root, exclude,
backup_logs, external_dir_num, location);
pgFileFree(file);
}
#define CHECK_FALSE 0
#define CHECK_TRUE 1
#define CHECK_EXCLUDE_FALSE 2
@@ -386,6 +424,7 @@ static char
dir_check_file(pgFile *file, bool backup_logs)
{
int i;
int sscanf_res;
bool in_tablespace = false;
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path);
@@ -433,6 +472,54 @@ dir_check_file(pgFile *file, bool backup_logs)
}
}
/*
* Do not copy tablespaces twice. It may happen if the tablespace is located
* inside the PGDATA.
*/
if (file->kind == PIO_KIND_DIRECTORY &&
strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
{
Oid tblspcOid;
char tmp_rel_path[MAXPGPATH];
/*
* Valid path for the tablespace is
* pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
*/
if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
return CHECK_FALSE;
sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s",
&tblspcOid, tmp_rel_path);
if (sscanf_res == 0)
return CHECK_FALSE;
}
if (in_tablespace)
{
char tmp_rel_path[MAXPGPATH];
sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
&(file->tblspcOid), tmp_rel_path,
&(file->dbOid));
/*
* We should skip other files and directories rather than
* TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace.
*/
if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0)
return CHECK_FALSE;
}
else if (path_is_prefix_of_path("global", file->rel_path))
{
file->tblspcOid = GLOBALTABLESPACE_OID;
}
else if (path_is_prefix_of_path("base", file->rel_path))
{
file->tblspcOid = DEFAULTTABLESPACE_OID;
sscanf(file->rel_path, "base/%u/", &(file->dbOid));
}
/* Do not backup ptrack_init files */
if (file->kind == PIO_KIND_REGULAR && strcmp(file->name, "ptrack_init") == 0)
return CHECK_FALSE;
@@ -469,79 +556,118 @@ dir_check_file(pgFile *file, bool backup_logs)
}
/*
* Excluding default files from the files list.
* Input:
* parray *files - an array of pgFile* to filter.
* croterion_fn - a callback that filters things out
* Output:
* true - if the file must be deleted from the list
* false - otherwise
* List files in parent->path directory. If "exclude" is true do not add into
* "files" files from pgdata_exclude_files and directories from
* pgdata_exclude_dir.
*
* TODO: should we check for interrupt here ?
*/
static void
dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool backup_logs,
int external_dir_num, fio_location location)
{
DIR *dir;
struct dirent *dent;
pioDrive_i drive;
static bool
exclude_files_cb(void *value, void *exclude_args) {
pgFile *file = (pgFile*) value;
exclude_cb_ctx *ex_ctx = (exclude_cb_ctx*) exclude_args;
if (parent->kind != PIO_KIND_DIRECTORY)
elog(ERROR, "\"%s\" is not a directory", parent_dir);
/*
* Check the file relative path for previously excluded dir prefix. These files
* should not be in the list, only their empty parent directory, see dir_check_file.
*
* Assuming that the excluded dir is ALWAYS followed by its content like this:
* pref/dir/
* pref/dir/file1
* pref/dir/file2
* pref/dir/file3
* ...
* we can make prefix checks only for files that subsequently follow the excluded dir
* and avoid unnecessary checks for the rest of the files. So we store the prefix length,
* update it and the prefix itself once we've got a CHECK_EXCLUDE_FALSE status code,
* keep doing prefix checks while there are files in that directory and set prefix length
* to 0 once they are gone.
*/
if(ex_ctx->pref_len > 0
&& strncmp(ex_ctx->exclude_dir_content_pref, file->rel_path, ex_ctx->pref_len) == 0) {
return true;
} else {
memset(ex_ctx->exclude_dir_content_pref, 0, ex_ctx->pref_len);
ex_ctx->pref_len = 0;
drive = pioDriveForLocation(location);
/* Open directory and list contents */
dir = fio_opendir(location, parent_dir);
if (dir == NULL)
{
if (errno == ENOENT)
{
/* Maybe the directory was removed */
return;
}
elog(ERROR, "Cannot open directory \"%s\": %s",
parent_dir, strerror(errno));
}
int check_res = dir_check_file(file, ex_ctx->backup_logs);
errno = 0;
while ((dent = fio_readdir(dir)))
{
pgFile *file;
char child[MAXPGPATH];
char rel_child[MAXPGPATH];
char check_res;
switch(check_res) {
case CHECK_FALSE:
return true;
break;
case CHECK_TRUE:;
return false;
break;
case CHECK_EXCLUDE_FALSE:
// since the excluded dir always goes before its contents, memorize it
// and use it for further files filtering.
strcpy(ex_ctx->exclude_dir_content_pref, file->rel_path);
ex_ctx->pref_len = strlen(file->rel_path);
return false;
break;
default:
// Should not get there normally.
assert(false);
return false;
break;
join_path_components(child, parent_dir, dent->d_name);
join_path_components(rel_child, parent->rel_path, dent->d_name);
file = pgFileNew(child, rel_child, true, external_dir_num,
drive);
if (file == NULL)
continue;
/* Skip entries point current dir or parent dir */
if (file->kind == PIO_KIND_DIRECTORY &&
(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
{
pgFileFree(file);
continue;
}
/* skip hidden files and directories */
if (file->name[0] == '.')
{
elog(WARNING, "Skip hidden file: '%s'", child);
pgFileFree(file);
continue;
}
/*
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR)
{
elog(WARNING, "Skip '%s': unexpected file format", child);
pgFileFree(file);
continue;
}
if (exclude)
{
check_res = dir_check_file(file, backup_logs);
if (check_res == CHECK_FALSE)
{
/* Skip */
pgFileFree(file);
continue;
}
else if (check_res == CHECK_EXCLUDE_FALSE)
{
/* We add the directory itself which content was excluded */
parray_append(files, file);
continue;
}
}
parray_append(files, file);
/*
* If the entry is a directory call dir_list_file_internal()
* recursively.
*/
if (file->kind == PIO_KIND_DIRECTORY)
dir_list_file_internal(files, file, child, exclude,
backup_logs, external_dir_num, location);
}
// Should not get there as well.
return false;
}
void exclude_files(parray *files, bool backup_logs) {
exclude_cb_ctx ctx = {
.pref_len = 0,
.backup_logs = backup_logs,
.exclude_dir_content_pref = "\0",
};
parray_remove_if(files, exclude_files_cb, (void*)&ctx, pgFileFree);
if (errno && errno != ENOENT)
{
int errno_tmp = errno;
fio_closedir(dir);
elog(ERROR, "Cannot read directory \"%s\": %s",
parent_dir, strerror(errno_tmp));
}
fio_closedir(dir);
}
/*
src/pg_probackup.h
+4 -1
View File
@@ -966,6 +966,10 @@ extern ft_str_t pb_control_line_get_str(pb_control_line *pb_line, const char *na
extern bool pb_control_line_try_int64(pb_control_line *pb_line, const char *name, int64 *value);
extern bool pb_control_line_try_str(pb_control_line *pb_line, const char *name, ft_str_t *value);
extern void db_list_dir(parray *files, const char *root, bool exclude,
bool backup_logs,
int external_dir_num, fio_location location);
extern const char *get_tablespace_mapping(const char *dir);
extern void create_data_directories(parray *dest_files,
const char *data_dir,
@@ -1019,7 +1023,6 @@ extern int pgPrefixCompareString(const void *str1, const void *str2);
extern int pgCompareOid(const void *f1, const void *f2);
extern void pfilearray_clear_locks(parray *file_list);
extern bool set_forkname(pgFile *file);
extern void exclude_files(parray *files, bool backup_logs);
/* in data.c */
extern bool check_data_file(pgFile *file, const char *from_fullpath, uint32 checksum_version);
src/restore.c
+3 -2
View File
@@ -871,7 +871,7 @@ restore_chain(InstanceState *instanceState,
elog(INFO, "Extracting the content of destination directory for incremental restore");
time(&start_time);
db_list_dir(pgdata_files, pgdata_path, false, false, 0);
db_list_dir(pgdata_files, pgdata_path, false, false, 0, FIO_DB_HOST);
/*
* TODO:
@@ -891,7 +891,8 @@ restore_chain(InstanceState *instanceState,
char *external_path = parray_get(external_dirs, i);
parray *external_files = parray_new();
db_list_dir(external_files, external_path, false, false, i+1);
db_list_dir(external_files, external_path,
false, false, i + 1, FIO_DB_HOST);
parray_concat(pgdata_files, external_files);
parray_free(external_files);
src/utils/file.c
-381
View File
@@ -5,10 +5,6 @@
#include <signal.h>
#include "file.h"
#include "catalog/pg_tablespace.h"
#if PG_VERSION_NUM < 110000
#include "catalog/catalog.h"
#endif
#include "storage/checksum.h"
#define PRINTF_BUF_SIZE 1024
@@ -39,16 +35,6 @@ typedef struct
int path_len;
} fio_send_request;
typedef struct
{
char path[MAXPGPATH];
bool handle_tablespaces;
bool follow_symlink;
bool backup_logs;
bool skip_hidden;
int external_dir_num;
} fio_list_dir_request;
typedef struct {
char path[MAXPGPATH];
bool root_as_well;
@@ -115,13 +101,6 @@ struct __attribute__((packed)) fio_req_open_write {
#undef fopen
#endif
static void dir_list_file(parray *files, const char *root, bool handle_tablespaces,
bool follow_symlink, bool backup_logs, bool skip_hidden,
int external_dir_num, pioDBDrive_i drive);
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool handle_tablespaces, bool follow_symlink, bool backup_logs,
bool skip_hidden, int external_dir_num, pioDBDrive_i drive);
void
setMyLocation(ProbackupSubcmd const subcmd)
{
@@ -1798,14 +1777,6 @@ pgFileGetCRC32(const char *file_path, bool missing_ok)
}
#endif /* PG_VERSION_NUM < 120000 */
void db_list_dir(parray *files, const char *root, bool handle_tablespaces,
bool backup_logs, int external_dir_num) {
pioDrive_i drive = pioDriveForLocation(FIO_DB_HOST);
$i(pioListDir, drive, .files = files, .root = root, .handle_tablespaces = handle_tablespaces,
.symlink_and_hidden = true, .backup_logs = backup_logs, .skip_hidden = true,
.external_dir_num = external_dir_num);
}
/*
* WARNING! this function is not paired with fio_remove_dir
* because there is no such function. Instead, it is paired
@@ -1826,272 +1797,6 @@ fio_remove_dir_impl(int out, char* buf) {
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
/*
* List files, symbolic links and directories in the directory "root" and add
* pgFile objects to "files". We add "root" to "files" if add_root is true.
*
* When follow_symlink is true, symbolic link is ignored and only file or
* directory linked to will be listed.
*
* TODO: make it strictly local
*/
static void
dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool follow_symlink,
bool backup_logs, bool skip_hidden, int external_dir_num, pioDBDrive_i drive)
{
pgFile *file;
Assert(!$i(pioIsRemote, drive));
file = pgFileNew(root, "", follow_symlink, external_dir_num,
$reduce(pioDrive, drive));
if (file == NULL)
{
/* For external directory this is not ok */
if (external_dir_num > 0)
elog(ERROR, "External directory is not found: \"%s\"", root);
else
return;
}
if (file->kind != PIO_KIND_DIRECTORY)
{
if (external_dir_num > 0)
elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected",
root);
else
elog(WARNING, "Skip \"%s\": unexpected file format", root);
return;
}
dir_list_file_internal(files, file, root, handle_tablespaces, follow_symlink,
backup_logs, skip_hidden, external_dir_num, drive);
pgFileFree(file);
}
/*
* List files in parent->path directory.
* If "handle_tablespaces" is true, handle recursive tablespaces
* and the ones located inside pgdata.
* If "follow_symlink" is true, follow symlinks so that the
* fio_stat call fetches the info from the file pointed to by the
* symlink, not from the symlink itself.
*
* TODO: should we check for interrupt here ?
*/
static void
dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool handle_tablespaces, bool follow_symlink, bool backup_logs,
bool skip_hidden, int external_dir_num, pioDBDrive_i drive)
{
DIR *dir;
struct dirent *dent;
bool in_tablespace = false;
Assert(!$i(pioIsRemote, drive));
if (parent->kind != PIO_KIND_DIRECTORY)
elog(ERROR, "\"%s\" is not a directory", parent_dir);
in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path);
/* Open directory and list contents */
dir = opendir(parent_dir);
if (dir == NULL)
{
if (errno == ENOENT)
{
/* Maybe the directory was removed */
return;
}
elog(ERROR, "Cannot open directory \"%s\": %s",
parent_dir, strerror(errno));
}
errno = 0;
while ((dent = readdir(dir)))
{
pgFile *file;
char child[MAXPGPATH];
char rel_child[MAXPGPATH];
join_path_components(child, parent_dir, dent->d_name);
join_path_components(rel_child, parent->rel_path, dent->d_name);
file = pgFileNew(child, rel_child, follow_symlink,
external_dir_num, $reduce(pioDrive, drive));
if (file == NULL)
continue;
/* Skip entries point current dir or parent dir */
if (file->kind == PIO_KIND_DIRECTORY &&
(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
{
pgFileFree(file);
continue;
}
/* skip hidden files and directories */
if (skip_hidden && file->name[0] == '.') {
elog(WARNING, "Skip hidden file: '%s'", child);
pgFileFree(file);
continue;
}
/*
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR)
{
elog(WARNING, "Skip '%s': unexpected file format", child);
pgFileFree(file);
continue;
}
if(handle_tablespaces) {
/*
* Do not copy tablespaces twice. It may happen if the tablespace is located
* inside the PGDATA.
*/
if (file->kind == PIO_KIND_DIRECTORY &&
strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
{
Oid tblspcOid;
char tmp_rel_path[MAXPGPATH];
int sscanf_res;
/*
* Valid path for the tablespace is
* pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
*/
if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path))
continue;
sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s",
&tblspcOid, tmp_rel_path);
if (sscanf_res == 0)
continue;
}
if (in_tablespace) {
char tmp_rel_path[MAXPGPATH];
ssize_t sscanf_res;
sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
&(file->tblspcOid), tmp_rel_path,
&(file->dbOid));
/*
* We should skip other files and directories rather than
* TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace.
*/
if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0)
continue;
} else if (path_is_prefix_of_path("global", file->rel_path)) {
file->tblspcOid = GLOBALTABLESPACE_OID;
} else if (path_is_prefix_of_path("base", file->rel_path)) {
file->tblspcOid = DEFAULTTABLESPACE_OID;
sscanf(file->rel_path, "base/%u/", &(file->dbOid));
}
}
parray_append(files, file);
/*
* If the entry is a directory call dir_list_file_internal()
* recursively.
*/
if (file->kind == PIO_KIND_DIRECTORY)
dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink,
backup_logs, skip_hidden, external_dir_num, drive);
}
if (errno && errno != ENOENT)
{
int errno_tmp = errno;
closedir(dir);
elog(ERROR, "Cannot read directory \"%s\": %s",
parent_dir, strerror(errno_tmp));
}
closedir(dir);
}
/*
* To get the arrays of files we use the same function dir_list_file(),
* that is used for local backup.
* After that we iterate over arrays and for every file send at least
* two messages to main process:
* 1. rel_path
* 2. metainformation (size, mtime, etc)
* 3. link path (optional)
*
* TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages
*/
static void
fio_list_dir_impl(int out, char* buf, pioDBDrive_i drive)
{
int i;
fio_header hdr;
fio_list_dir_request *req = (fio_list_dir_request*) buf;
parray *file_files = parray_new();
/*
* Disable logging into console any messages with exception of ERROR messages,
* because currently we have no mechanism to notify the main process
* about then message been sent.
* TODO: correctly send elog messages from agent to main process.
*/
instance_config.logger.log_level_console = ERROR;
dir_list_file(file_files, req->path, req->handle_tablespaces,
req->follow_symlink, req->backup_logs, req->skip_hidden,
req->external_dir_num, drive);
/* send information about files to the main process */
for (i = 0; i < parray_num(file_files); i++)
{
fio_pgFile fio_file;
pgFile *file = (pgFile *) parray_get(file_files, i);
fio_file.kind = file->kind;
fio_file.mode = file->mode;
fio_file.size = file->size;
fio_file.mtime = file->mtime;
fio_file.is_datafile = file->is_datafile;
fio_file.tblspcOid = file->tblspcOid;
fio_file.dbOid = file->dbOid;
fio_file.relOid = file->relOid;
fio_file.forkName = file->forkName;
fio_file.segno = file->segno;
fio_file.external_dir_num = file->external_dir_num;
if (file->linked)
fio_file.linked_len = strlen(file->linked) + 1;
else
fio_file.linked_len = 0;
hdr.cop = FIO_SEND_FILE;
hdr.size = strlen(file->rel_path) + 1;
/* send rel_path first */
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(out, file->rel_path, hdr.size), hdr.size);
/* now send file metainformation */
IO_CHECK(fio_write_all(out, &fio_file, sizeof(fio_file)), sizeof(fio_file));
/* If file is a symlink, then send link path */
if (file->linked)
IO_CHECK(fio_write_all(out, file->linked, fio_file.linked_len), fio_file.linked_len);
pgFileFree(file);
}
parray_free(file_files);
hdr = (fio_header){.cop = FIO_SEND_FILE_EOF};
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
PageState *
fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version,
@@ -2515,9 +2220,6 @@ fio_communicate(int in, int out)
case FIO_SEEK: /* Set current position in file */
fio_seek_impl(fd[hdr.handle], hdr.arg);
break;
case FIO_LIST_DIR:
fio_list_dir_impl(out, buf, drive);
break;
case FIO_REMOVE_DIR:
fio_remove_dir_impl(out, buf);
break;
@@ -3306,16 +3008,6 @@ pioLocalDrive_pioMakeDir(VSelf, path_t path, mode_t mode, bool strict)
return $syserr(errno, "Cannot make dir {path:q}", path(path));
}
static void
pioLocalDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_tablespaces,
bool follow_symlink, bool backup_logs, bool skip_hidden,
int external_dir_num) {
FOBJ_FUNC_ARP();
Self(pioLocalDrive);
dir_list_file(files, root, handle_tablespaces, follow_symlink, backup_logs,
skip_hidden, external_dir_num, $bind(pioDBDrive, self));
}
static pioDirIter_i
pioLocalDrive_pioOpenDir(VSelf, path_t path, err_i* err)
{
@@ -4123,79 +3815,6 @@ pioRemoteDrive_pioMakeDir(VSelf, path_t path, mode_t mode, bool strict)
return $syserr(hdr.arg, "Cannot make dir {path:q}", path(path));
}
static void
pioRemoteDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_tablespaces,
bool follow_symlink, bool backup_logs, bool skip_hidden,
int external_dir_num) {
FOBJ_FUNC_ARP();
fio_header hdr;
fio_list_dir_request req;
char *buf = pgut_malloc(CHUNK_SIZE);
/* Send to the agent message with parameters for directory listing */
memset(&req, 0, sizeof(req));
snprintf(req.path, MAXPGPATH, "%s", root);
req.handle_tablespaces = handle_tablespaces;
req.follow_symlink = follow_symlink;
req.backup_logs = backup_logs;
req.skip_hidden = skip_hidden;
req.external_dir_num = external_dir_num;
hdr = (fio_header){.cop = FIO_LIST_DIR, .size=sizeof(req)};
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, &req, hdr.size), hdr.size);
for (;;) {
/* receive data */
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
if (hdr.cop == FIO_SEND_FILE_EOF) {
/* the work is done */
break;
} else if (hdr.cop == FIO_SEND_FILE) {
pgFile *file = NULL;
fio_pgFile fio_file;
/* receive rel_path */
IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size);
file = pgFileInit(buf);
/* receive metainformation */
IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file));
file->kind = fio_file.kind;
file->mode = fio_file.mode;
file->size = fio_file.size;
file->mtime = fio_file.mtime;
file->is_datafile = fio_file.is_datafile;
file->tblspcOid = fio_file.tblspcOid;
file->dbOid = fio_file.dbOid;
file->relOid = fio_file.relOid;
file->forkName = fio_file.forkName;
file->segno = fio_file.segno;
file->external_dir_num = fio_file.external_dir_num;
if (fio_file.linked_len > 0) {
IO_CHECK(fio_read_all(fio_stdin, buf, fio_file.linked_len), fio_file.linked_len);
file->linked = pgut_malloc(fio_file.linked_len);
snprintf(file->linked, fio_file.linked_len, "%s", buf);
}
// elog(INFO, "Received file: %s, mode: %u, size: %lu, mtime: %lu",
// file->rel_path, file->mode, file->size, file->mtime);
parray_append(files, file);
} else {
/* TODO: fio_disconnect may get assert fail when running after this */
elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop);
}
}
pg_free(buf);
}
static pioDirIter_i
pioRemoteDrive_pioOpenDir(VSelf, path_t path, err_i* err)
{
src/utils/file.h
+1 -8
View File
@@ -178,9 +178,6 @@ extern int fio_remove(fio_location location, const char* path, bool missing_
extern ssize_t fio_readlink(fio_location location, const char *path, char *value, size_t valsiz);
extern pid_t fio_check_postmaster(fio_location location, const char *pgdata);
extern void db_list_dir(parray *files, const char *root, bool handle_tablespaces,
bool backup_logs, int external_dir_num);
extern PageState *fio_get_checksum_map(fio_location location, const char *fullpath, uint32 checksum_version,
int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno);
struct datapagemap; /* defined in datapagemap.h */
@@ -299,9 +296,6 @@ fobj_iface(pioPagesIterator);
#define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict)
#define mth__pioOpenDir pioDirIter_i, (path_t, path), (err_i*, err)
#define mth__pioIsDirEmpty bool, (path_t, path), (err_i*, err)
#define mth__pioListDir void, (parray *, files), (const char *, root), \
(bool, handle_tablespaces), (bool, symlink_and_hidden), \
(bool, backup_logs), (bool, skip_hidden), (int, external_dir_num)
#define mth__pioRemoveDir void, (const char *, root), (bool, root_as_well)
/* pioReadFile and pioWriteFile should be used only for small files */
#define PIO_READ_WRITE_FILE_LIMIT (16*1024*1024)
@@ -329,7 +323,6 @@ fobj_method(pioMakeDir);
fobj_method(pioFilesAreSame);
fobj_method(pioOpenDir);
fobj_method(pioIsDirEmpty);
fobj_method(pioListDir);
fobj_method(pioRemoveDir);
fobj_method(pioReadFile);
fobj_method(pioWriteFile);
@@ -338,7 +331,7 @@ fobj_method(pioIteratePages);
#define iface__pioDrive mth(pioOpenRead, pioOpenReadStream), \
mth(pioStat, pioRemove), \
mth(pioExists, pioGetCRC32, pioIsRemote), \
mth(pioMakeDir, pioOpenDir, pioIsDirEmpty, pioListDir, pioRemoveDir), \
mth(pioMakeDir, pioOpenDir, pioIsDirEmpty, pioRemoveDir), \
mth(pioFilesAreSame, pioReadFile, pioWriteFile), \
mth(pioOpenRewrite)
fobj_iface(pioDrive);