diff --git a/src/dir.c b/src/dir.c index a5462d38..29003d88 100644 --- a/src/dir.c +++ b/src/dir.c @@ -16,7 +16,6 @@ #if PG_VERSION_NUM < 110000 #include "catalog/catalog.h" #endif -#include "catalog/pg_tablespace.h" #include #include @@ -125,9 +124,6 @@ typedef struct exclude_cb_ctx { static char dir_check_file(pgFile *file, bool backup_logs); -static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool handle_tablespaces, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); static void opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, const char *type); static void cleanup_tablespace(const char *path); @@ -371,47 +367,6 @@ db_map_entry_free(void *entry) free(entry); } -/* - * List files, symbolic links and directories in the directory "root" and add - * pgFile objects to "files". We add "root" to "files" if add_root is true. - * - * When follow_symlink is true, symbolic link is ignored and only file or - * directory linked to will be listed. - * - * TODO: make it strictly local - */ -void -dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool follow_symlink, - bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location) -{ - pgFile *file; - - file = pgFileNew(root, "", follow_symlink, external_dir_num, location); - if (file == NULL) - { - /* For external directory this is not ok */ - if (external_dir_num > 0) - elog(ERROR, "External directory is not found: \"%s\"", root); - else - return; - } - - if (file->kind != PIO_KIND_DIRECTORY) - { - if (external_dir_num > 0) - elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", - root); - else - elog(WARNING, "Skip \"%s\": unexpected file format", root); - return; - } - - dir_list_file_internal(files, file, root, handle_tablespaces, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); - - pgFileFree(file); -} - #define CHECK_FALSE 0 #define CHECK_TRUE 1 #define CHECK_EXCLUDE_FALSE 2 @@ -591,151 +546,6 @@ void exclude_files(parray *files, bool backup_logs) { parray_remove_if(files, exclude_files_cb, (void*)&ctx, pgFileFree); } -/* - * List files in parent->path directory. - * If "handle_tablespaces" is true, handle recursive tablespaces - * and the ones located inside pgdata. - * If "follow_symlink" is true, follow symlinks so that the - * fio_stat call fetches the info from the file pointed to by the - * symlink, not from the symlink itself. - * - * TODO: should we check for interrupt here ? - */ -static void -dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool handle_tablespaces, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location) -{ - DIR *dir; - struct dirent *dent; - bool in_tablespace = false; - - if (parent->kind != PIO_KIND_DIRECTORY) - elog(ERROR, "\"%s\" is not a directory", parent_dir); - - in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path); - - /* Open directory and list contents */ - dir = fio_opendir(location, parent_dir); - if (dir == NULL) - { - if (errno == ENOENT) - { - /* Maybe the directory was removed */ - return; - } - elog(ERROR, "Cannot open directory \"%s\": %s", - parent_dir, strerror(errno)); - } - - errno = 0; - while ((dent = fio_readdir(dir))) - { - pgFile *file; - char child[MAXPGPATH]; - char rel_child[MAXPGPATH]; - - join_path_components(child, parent_dir, dent->d_name); - join_path_components(rel_child, parent->rel_path, dent->d_name); - - file = pgFileNew(child, rel_child, follow_symlink, - external_dir_num, location); - if (file == NULL) - continue; - - /* Skip entries point current dir or parent dir */ - if (file->kind == PIO_KIND_DIRECTORY && - (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) - { - pgFileFree(file); - continue; - } - - /* skip hidden files and directories */ - if (skip_hidden && file->name[0] == '.') { - elog(WARNING, "Skip hidden file: '%s'", child); - pgFileFree(file); - continue; - } - - /* - * Add only files, directories and links. Skip sockets and other - * unexpected file formats. - */ - if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR) - { - elog(WARNING, "Skip '%s': unexpected file format", child); - pgFileFree(file); - continue; - } - - if(handle_tablespaces) { - /* - * Do not copy tablespaces twice. It may happen if the tablespace is located - * inside the PGDATA. - */ - if (file->kind == PIO_KIND_DIRECTORY && - strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) - { - Oid tblspcOid; - char tmp_rel_path[MAXPGPATH]; - int sscanf_res; - - /* - * Valid path for the tablespace is - * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY - */ - if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) - continue; - sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s", - &tblspcOid, tmp_rel_path); - if (sscanf_res == 0) - continue; - } - - if (in_tablespace) { - char tmp_rel_path[MAXPGPATH]; - ssize_t sscanf_res; - - sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", - &(file->tblspcOid), tmp_rel_path, - &(file->dbOid)); - - /* - * We should skip other files and directories rather than - * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace. - */ - if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) - continue; - } else if (path_is_prefix_of_path("global", file->rel_path)) { - file->tblspcOid = GLOBALTABLESPACE_OID; - } else if (path_is_prefix_of_path("base", file->rel_path)) { - file->tblspcOid = DEFAULTTABLESPACE_OID; - sscanf(file->rel_path, "base/%u/", &(file->dbOid)); - } - } - - parray_append(files, file); - - /* - * If the entry is a directory call dir_list_file_internal() - * recursively. - */ - if (file->kind == PIO_KIND_DIRECTORY) - dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); - } - - if (errno && errno != ENOENT) - { - int errno_tmp = errno; - fio_closedir(dir); - elog(ERROR, "Cannot read directory \"%s\": %s", - parent_dir, strerror(errno_tmp)); - } - fio_closedir(dir); -} - /* * Retrieve tablespace path, either relocated or original depending on whether * -T was passed or not. diff --git a/src/pg_probackup.h b/src/pg_probackup.h index 73efb5d8..7df0d04e 100644 --- a/src/pg_probackup.h +++ b/src/pg_probackup.h @@ -995,9 +995,6 @@ extern const char* deparse_compress_alg(int alg); extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory); extern bool get_control_value_str(const char *str, const char *name, char *value_str, size_t value_str_size, bool is_mandatory); -extern void dir_list_file(parray *files, const char *root, bool handle_tablespaces, - bool follow_symlink, bool backup_logs, bool skip_hidden, - int external_dir_num, fio_location location); extern const char *get_tablespace_mapping(const char *dir); extern void create_data_directories(parray *dest_files, diff --git a/src/utils/file.c b/src/utils/file.c index 0c51c423..4d4bae88 100644 --- a/src/utils/file.c +++ b/src/utils/file.c @@ -5,6 +5,7 @@ #include #include "file.h" +#include "catalog/pg_tablespace.h" #include "storage/checksum.h" #define PRINTF_BUF_SIZE 1024 @@ -90,6 +91,13 @@ typedef struct #undef fopen #endif +static void dir_list_file(parray *files, const char *root, bool handle_tablespaces, + bool follow_symlink, bool backup_logs, bool skip_hidden, + int external_dir_num, fio_location location); +static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, + bool handle_tablespaces, bool follow_symlink, bool backup_logs, + bool skip_hidden, int external_dir_num, fio_location location); + void setMyLocation(ProbackupSubcmd const subcmd) { @@ -3271,6 +3279,192 @@ fio_remove_dir_impl(int out, char* buf) { IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); } +/* + * List files, symbolic links and directories in the directory "root" and add + * pgFile objects to "files". We add "root" to "files" if add_root is true. + * + * When follow_symlink is true, symbolic link is ignored and only file or + * directory linked to will be listed. + * + * TODO: make it strictly local + */ +static void +dir_list_file(parray *files, const char *root, bool handle_tablespaces, bool follow_symlink, + bool backup_logs, bool skip_hidden, int external_dir_num, fio_location location) +{ + pgFile *file; + + file = pgFileNew(root, "", follow_symlink, external_dir_num, location); + if (file == NULL) + { + /* For external directory this is not ok */ + if (external_dir_num > 0) + elog(ERROR, "External directory is not found: \"%s\"", root); + else + return; + } + + if (file->kind != PIO_KIND_DIRECTORY) + { + if (external_dir_num > 0) + elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", + root); + else + elog(WARNING, "Skip \"%s\": unexpected file format", root); + return; + } + + dir_list_file_internal(files, file, root, handle_tablespaces, follow_symlink, + backup_logs, skip_hidden, external_dir_num, location); + + pgFileFree(file); +} + +/* + * List files in parent->path directory. + * If "handle_tablespaces" is true, handle recursive tablespaces + * and the ones located inside pgdata. + * If "follow_symlink" is true, follow symlinks so that the + * fio_stat call fetches the info from the file pointed to by the + * symlink, not from the symlink itself. + * + * TODO: should we check for interrupt here ? + */ +static void +dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, + bool handle_tablespaces, bool follow_symlink, bool backup_logs, + bool skip_hidden, int external_dir_num, fio_location location) +{ + DIR *dir; + struct dirent *dent; + bool in_tablespace = false; + + if (parent->kind != PIO_KIND_DIRECTORY) + elog(ERROR, "\"%s\" is not a directory", parent_dir); + + in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, parent->rel_path); + + /* Open directory and list contents */ + dir = fio_opendir(location, parent_dir); + if (dir == NULL) + { + if (errno == ENOENT) + { + /* Maybe the directory was removed */ + return; + } + elog(ERROR, "Cannot open directory \"%s\": %s", + parent_dir, strerror(errno)); + } + + errno = 0; + while ((dent = fio_readdir(dir))) + { + pgFile *file; + char child[MAXPGPATH]; + char rel_child[MAXPGPATH]; + + join_path_components(child, parent_dir, dent->d_name); + join_path_components(rel_child, parent->rel_path, dent->d_name); + + file = pgFileNew(child, rel_child, follow_symlink, + external_dir_num, location); + if (file == NULL) + continue; + + /* Skip entries point current dir or parent dir */ + if (file->kind == PIO_KIND_DIRECTORY && + (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) + { + pgFileFree(file); + continue; + } + + /* skip hidden files and directories */ + if (skip_hidden && file->name[0] == '.') { + elog(WARNING, "Skip hidden file: '%s'", child); + pgFileFree(file); + continue; + } + + /* + * Add only files, directories and links. Skip sockets and other + * unexpected file formats. + */ + if (file->kind != PIO_KIND_DIRECTORY && file->kind != PIO_KIND_REGULAR) + { + elog(WARNING, "Skip '%s': unexpected file format", child); + pgFileFree(file); + continue; + } + + if(handle_tablespaces) { + /* + * Do not copy tablespaces twice. It may happen if the tablespace is located + * inside the PGDATA. + */ + if (file->kind == PIO_KIND_DIRECTORY && + strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) + { + Oid tblspcOid; + char tmp_rel_path[MAXPGPATH]; + int sscanf_res; + + /* + * Valid path for the tablespace is + * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY + */ + if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) + continue; + sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s", + &tblspcOid, tmp_rel_path); + if (sscanf_res == 0) + continue; + } + + if (in_tablespace) { + char tmp_rel_path[MAXPGPATH]; + ssize_t sscanf_res; + + sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", + &(file->tblspcOid), tmp_rel_path, + &(file->dbOid)); + + /* + * We should skip other files and directories rather than + * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace. + */ + if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) + continue; + } else if (path_is_prefix_of_path("global", file->rel_path)) { + file->tblspcOid = GLOBALTABLESPACE_OID; + } else if (path_is_prefix_of_path("base", file->rel_path)) { + file->tblspcOid = DEFAULTTABLESPACE_OID; + sscanf(file->rel_path, "base/%u/", &(file->dbOid)); + } + } + + parray_append(files, file); + + /* + * If the entry is a directory call dir_list_file_internal() + * recursively. + */ + if (file->kind == PIO_KIND_DIRECTORY) + dir_list_file_internal(files, file, child, handle_tablespaces, follow_symlink, + backup_logs, skip_hidden, external_dir_num, location); + } + + if (errno && errno != ENOENT) + { + int errno_tmp = errno; + fio_closedir(dir); + elog(ERROR, "Cannot read directory \"%s\": %s", + parent_dir, strerror(errno_tmp)); + } + fio_closedir(dir); +} + /* * To get the arrays of files we use the same function dir_list_file(), * that is used for local backup.