db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-02-13 14:58:35 +02:00
Code Issues Releases Activity

Merge branch 'master' into release_2_4

Browse Source
This commit is contained in:
Grigory Smolkin 2020-05-27 01:15:41 +03:00
commit df1e65802a
13 changed files with 83 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -54,7 +54,7 @@ Regardless of the chosen backup type, all backups taken with `pg_probackup` supp
## Current release
[2.3.3](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.3)
[2.3.4](https://github.com/postgrespro/pg_probackup/releases/tag/2.3.4)
## Documentation

2
src/archive.c
View File

@ -944,7 +944,7 @@ setup_push_filelist(const char *archive_status_dir, const char *first_file,
/* get list of files from archive_status */
status_files = parray_new();
dir_list_file(status_files, archive_status_dir, false, false, false, false, 0, FIO_DB_HOST);
dir_list_file(status_files, archive_status_dir, false, false, false, false, true, 0, FIO_DB_HOST);
parray_qsort(status_files, pgFileCompareName);
for (i = 0; i < parray_num(status_files); i++)

10
src/backup.c
View File

@ -333,10 +333,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
/* list files with the logical path. omit $PGDATA */
if (fio_is_remote(FIO_DB_HOST))
fio_list_dir(backup_files_list, instance_config.pgdata,
true, true, false, backup_logs, 0);
true, true, false, backup_logs, true, 0);
else
dir_list_file(backup_files_list, instance_config.pgdata,
true, true, false, backup_logs, 0, FIO_LOCAL_HOST);
true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST);
/*
* Get database_map (name to oid) for use in partial restore feature.
@ -356,10 +356,10 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
* 0 value is not external dir */
if (fio_is_remote(FIO_DB_HOST))
fio_list_dir(backup_files_list, parray_get(external_dirs, i),
false, true, false, false, i+1);
false, true, false, false, true, i+1);
else
dir_list_file(backup_files_list, parray_get(external_dirs, i),
false, true, false, false, i+1, FIO_LOCAL_HOST);
false, true, false, false, true, i+1, FIO_LOCAL_HOST);
}
}
@ -615,7 +615,7 @@ do_backup_instance(PGconn *backup_conn, PGNodeInfo *nodeInfo, bool no_sync, bool
/* Scan backup PG_XLOG_DIR */
xlog_files_list = parray_new();
join_path_components(pg_xlog_path, database_path, PG_XLOG_DIR);
dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, 0,
dir_list_file(xlog_files_list, pg_xlog_path, false, true, false, false, true, 0,
FIO_BACKUP_HOST);
/* TODO: Drop streamed WAL segments greater than stop_lsn */

2
src/catalog.c
View File

@ -878,7 +878,7 @@ catalog_get_timelines(InstanceConfig *instance)
/* read all xlog files that belong to this archive */
sprintf(arclog_path, "%s/%s/%s", backup_path, "wal", instance->name);
dir_list_file(xlog_files_list, arclog_path, false, false, false, false, 0, FIO_BACKUP_HOST);
dir_list_file(xlog_files_list, arclog_path, false, false, false, false, true, 0, FIO_BACKUP_HOST);
parray_qsort(xlog_files_list, pgFileCompareName);
timelineinfos = parray_new();

2
src/checkdb.c
View File

@ -208,7 +208,7 @@ do_block_validation(char *pgdata, uint32 checksum_version)
/* list files with the logical path. omit $PGDATA */
dir_list_file(files_list, pgdata, true, true,
false, false, 0, FIO_DB_HOST);
false, false, true, 0, FIO_DB_HOST);
/*
* Sort pathname ascending.

2
src/delete.c
View File

@ -750,7 +750,7 @@ delete_backup_files(pgBackup *backup)
/* list files to be deleted */
files = parray_new();
dir_list_file(files, backup->root_dir, false, false, true, false, 0, FIO_LOCAL_HOST);
dir_list_file(files, backup->root_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST);
/* delete leaf node first */
parray_qsort(files, pgFileCompareRelPathWithExternalDesc);

14
src/dir.c
View File

@ -124,8 +124,9 @@ typedef struct TablespaceCreatedList
static int pgCompareString(const void *str1, const void *str2);
static char dir_check_file(pgFile *file);
static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool follow_symlink,
bool exclude, bool follow_symlink, bool skip_hidden,
int external_dir_num, fio_location location);
static void opt_path_map(ConfigOption *opt, const char *arg,
TablespaceList *list, const char *type);
@ -513,7 +514,8 @@ db_map_entry_free(void *entry)
*/
void
dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, bool backup_logs, int external_dir_num, fio_location location)
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num,
fio_location location)
{
pgFile *file;
@ -551,7 +553,7 @@ dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink
parray_append(files, file);
dir_list_file_internal(files, file, root, exclude, follow_symlink,
external_dir_num, location);
skip_hidden, external_dir_num, location);
if (!add_root)
pgFileFree(file);
@ -773,7 +775,7 @@ dir_check_file(pgFile *file)
*/
static void
dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
bool exclude, bool follow_symlink,
bool exclude, bool follow_symlink, bool skip_hidden,
int external_dir_num, fio_location location)
{
DIR *dir;
@ -820,7 +822,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
}
/* skip hidden files and directories */
if (file->name[0] == '.')
if (skip_hidden && file->name[0] == '.')
{
elog(WARNING, "Skip hidden file: '%s'", child);
pgFileFree(file);
@ -863,7 +865,7 @@ dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir,
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, file, child, exclude, follow_symlink,
external_dir_num, location);
skip_hidden, external_dir_num, location);
}
if (errno && errno != ENOENT)

2
src/merge.c
View File

@ -1062,7 +1062,7 @@ remove_dir_with_files(const char *path)
int i;
char full_path[MAXPGPATH];
dir_list_file(files, path, false, false, true, false, 0, FIO_LOCAL_HOST);
dir_list_file(files, path, false, false, true, false, false, 0, FIO_LOCAL_HOST);
parray_qsort(files, pgFileCompareRelPathWithExternalDesc);
for (i = 0; i < parray_num(files); i++)
{

15
src/pg_probackup.c
View File

@ -561,9 +561,18 @@ main(int argc, char *argv[])
setMyLocation();
}
/* disable logging into file for archive-push and archive-get */
if (backup_subcmd == ARCHIVE_GET_CMD ||
backup_subcmd == ARCHIVE_PUSH_CMD)
/*
* Disable logging into file for archive-push and archive-get.
* Note, that we should NOT use fio_is_remote() here,
* because it will launch ssh connection and we do not
* want it, because it will kill archive-get prefetch
* performance.
*
* TODO: make logging into file possible via ssh
*/
if (fio_is_remote_simple(FIO_BACKUP_HOST) &&
(backup_subcmd == ARCHIVE_GET_CMD ||
backup_subcmd == ARCHIVE_PUSH_CMD))
{
instance_config.logger.log_level_file = LOG_OFF;
}

9
src/pg_probackup.h
View File

@ -241,8 +241,8 @@ typedef enum ShowFormat
#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */
#define FILE_NOT_FOUND (-2) /* file disappeared during backup */
#define BLOCKNUM_INVALID (-1)
#define PROGRAM_VERSION "2.3.3"
#define AGENT_PROTOCOL_VERSION 20303
#define PROGRAM_VERSION "2.3.4"
#define AGENT_PROTOCOL_VERSION 20304
typedef struct ConnectionOptions
@ -860,7 +860,7 @@ extern const char* deparse_compress_alg(int alg);
/* in dir.c */
extern void dir_list_file(parray *files, const char *root, bool exclude,
bool follow_symlink, bool add_root, bool backup_logs,
int external_dir_num, fio_location location);
bool skip_hidden, int external_dir_num, fio_location location);
extern void create_data_directories(parray *dest_files,
const char *data_dir,
@ -1036,7 +1036,7 @@ extern int fio_send_file(const char *from_fullpath, const char *to_fullpath, FIL
pgFile *file, char **errormsg);
extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink,
bool add_root, bool backup_logs, int external_dir_num);
bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num);
extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
@ -1052,6 +1052,7 @@ extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict);
/* Check if specified location is local for current node */
extern bool fio_is_remote(fio_location location);
extern bool fio_is_remote_simple(fio_location location);
extern void get_header_errormsg(Page page, char **errormsg);
extern void get_checksum_errormsg(Page page, char **errormsg,

19
src/utils/file.c
View File

@ -44,6 +44,7 @@ typedef struct
bool add_root;
bool backup_logs;
bool exclusive_backup;
bool skip_hidden;
int external_dir_num;
} fio_list_dir_request;
@ -175,6 +176,15 @@ bool fio_is_remote(fio_location location)
return is_remote;
}
/* Check if specified location is local for current node */
bool fio_is_remote_simple(fio_location location)
{
bool is_remote = MyLocation != FIO_LOCAL_HOST
&& location != FIO_LOCAL_HOST
&& location != MyLocation;
return is_remote;
}
/* Try to read specified amount of bytes unless error or EOF are encountered */
static ssize_t fio_read_all(int fd, void* buf, size_t size)
{
@ -1536,7 +1546,7 @@ static void fio_send_pages_impl(int out, char* buf)
hdr.arg = OPEN_FAILED;
errormsg = pgut_malloc(ERRMSG_MAX_LEN);
/* Construct the error message */
snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open source file '%s': %s",
snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file \"%s\": %s",
from_fullpath, strerror(errno));
hdr.size = strlen(errormsg) + 1;
}
@ -2065,7 +2075,8 @@ cleanup:
/* Compile the array of files located on remote machine in directory root */
void fio_list_dir(parray *files, const char *root, bool exclude,
bool follow_symlink, bool add_root, bool backup_logs, int external_dir_num)
bool follow_symlink, bool add_root, bool backup_logs,
bool skip_hidden, int external_dir_num)
{
fio_header hdr;
fio_list_dir_request req;
@ -2078,6 +2089,7 @@ void fio_list_dir(parray *files, const char *root, bool exclude,
req.add_root = add_root;
req.backup_logs = backup_logs;
req.exclusive_backup = exclusive_backup;
req.skip_hidden = skip_hidden;
req.external_dir_num = external_dir_num;
hdr.cop = FIO_LIST_DIR;
@ -2172,7 +2184,8 @@ static void fio_list_dir_impl(int out, char* buf)
exclusive_backup = req->exclusive_backup;
dir_list_file(file_files, req->path, req->exclude, req->follow_symlink,
req->add_root, req->backup_logs, req->external_dir_num, FIO_LOCAL_HOST);
req->add_root, req->backup_logs, req->skip_hidden,
req->external_dir_num, FIO_LOCAL_HOST);
/* send information about files to the main process */
for (i = 0; i < parray_num(file_files); i++)

30
tests/compatibility.py
View File

@ -958,3 +958,33 @@ class CompatibilityTest(ProbackupTest, unittest.TestCase):
# Clean after yourself
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
def test_hidden_files(self):
"""
old_version should be < 2.3.0
Create hidden file in pgdata, take backup
with old binary, then try to delete backup
with new binary
"""
fname = self.id().split('.')[3]
backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
node = self.make_simple_node(
base_dir=os.path.join(module_name, fname, 'node'),
set_replication=True,
initdb_params=['--data-checksums'],
pg_options={'autovacuum': 'off'})
self.init_pb(backup_dir, old_binary=True)
self.add_instance(backup_dir, 'node', node, old_binary=True)
node.slow_start()
open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close()
backup_id = self.backup_node(
backup_dir, 'node',node, old_binary=True, options=['--stream'])
self.delete_pb(backup_dir, 'node', backup_id)
# Clean after yourself
self.del_test_dir(module_name, fname)

2
tests/expected/option_version.out
View File

@ -1 +1 @@
pg_probackup 2.3.3
pg_probackup 2.3.4