db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

Merge branch 'pgpro-1376'

Browse Source
This commit is contained in:
Grigory Smolkin
2018-04-28 18:46:28 +03:00
parent 8371dbfe19 3b3e55b267
commit def0ac28ed
6 changed files with 406 additions and 425 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/backup.c
+53 -228
View File
@@ -99,6 +99,7 @@ static void pg_switch_wal(PGconn *conn);
static void pg_stop_backup(pgBackup *backup);
static int checkpoint_timeout(void);
//static void backup_list_file(parray *files, const char *root, )
static void parse_backup_filelist_filenames(parray *files, const char *root);
static void write_backup_file_list(parray *files, const char *root);
static void wait_wal_lsn(XLogRecPtr lsn, bool wait_prev_segment);
@@ -273,7 +274,8 @@ ReceiveFileList(parray* files, PGconn *conn, PGresult *res, int rownum)
/* read one file via replication protocol
* and write it to the destination subdir in 'backup_path' */
static void remote_copy_file(PGconn *conn, pgFile* file)
static void
remote_copy_file(PGconn *conn, pgFile* file)
{
PGresult *res;
char *copybuf = NULL;
@@ -2095,256 +2097,75 @@ backup_files(void *arg)
static void
parse_backup_filelist_filenames(parray *files, const char *root)
{
size_t i;
Oid unlogged_file_reloid = 0;
size_t i = 0;
Oid unlogged_file_reloid = 0;
for (i = 0; i < parray_num(files); i++)
while (i < parray_num(files))
{
pgFile *file = (pgFile *) parray_get(files, i);
char *relative;
char filename[MAXPGPATH];
int sscanf_result;
relative = GetRelativePath(file->path, root);
filename[0] = '\0';
elog(VERBOSE, "-----------------------------------------------------: %s", relative);
if (path_is_prefix_of_path("global", relative))
if (S_ISREG(file->mode) &&
path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
{
file->tblspcOid = GLOBALTABLESPACE_OID;
sscanf_result = sscanf(relative, "global/%s", filename);
elog(VERBOSE, "global sscanf result: %i", sscanf_result);
if (strcmp(relative, "global") == 0)
/*
* Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
* Legal only in case of 'pg_compression'
*/
if (strcmp(file->name, "pg_compression") == 0)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "the global itself, filepath %s", relative);
file->is_database = true; /* TODO add an explanation */
}
else if (sscanf_result == 1)
{
elog(VERBOSE, "filename %s, filepath %s", filename, relative);
Oid tblspcOid;
Oid dbOid;
char tmp_rel_path[MAXPGPATH];
/*
* Check that the file is located under
* TABLESPACE_VERSION_DIRECTORY
*/
sscanf_result = sscanf(relative, PG_TBLSPC_DIR "/%u/%s/%u",
&tblspcOid, tmp_rel_path, &dbOid);
/* Yes, it is */
if (sscanf_result == 2 &&
strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0)
set_cfs_datafiles(files, root, relative, i);
}
}
else if (path_is_prefix_of_path("base", relative))
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
file->name && file->name[0])
{
file->tblspcOid = DEFAULTTABLESPACE_OID;
sscanf_result = sscanf(relative, "base/%u/%s", &(file->dbOid), filename);
elog(VERBOSE, "base sscanf result: %i", sscanf_result);
if (strcmp(relative, "base") == 0)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "the base itself, filepath %s", relative);
}
else if (sscanf_result == 1)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "dboid %u, filepath %s", file->dbOid, relative);
file->is_database = true;
}
else
{
elog(VERBOSE, "dboid %u, filename %s, filepath %s", file->dbOid, filename, relative);
}
}
else if (path_is_prefix_of_path(PG_TBLSPC_DIR, relative))
{
char temp_relative_path[MAXPGPATH];
sscanf_result = sscanf(relative, "pg_tblspc/%u/%s", &(file->tblspcOid), temp_relative_path);
elog(VERBOSE, "pg_tblspc sscanf result: %i", sscanf_result);
if (strcmp(relative, "pg_tblspc") == 0)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "the pg_tblspc itself, filepath %s", relative);
}
else if (sscanf_result == 1)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "tblspcOid %u, filepath %s", file->tblspcOid, relative);
}
else
{
/*continue parsing */
sscanf_result = sscanf(temp_relative_path+strlen(TABLESPACE_VERSION_DIRECTORY)+1, "%u/%s",
&(file->dbOid), filename);
elog(VERBOSE, "TABLESPACE_VERSION_DIRECTORY sscanf result: %i", sscanf_result);
if (sscanf_result == -1)
{
elog(VERBOSE, "The TABLESPACE_VERSION_DIRECTORY itself, filepath %s", relative);
}
else if (sscanf_result == 0)
{
/* Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
Legal only in case of 'pg_compression'
*/
if (strcmp(file->name, "pg_compression") == 0)
{
elog(VERBOSE, "Found pg_compression file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
/*Set every datafile in tablespace as is_cfs */
set_cfs_datafiles(files, root, relative, i);
}
else
{
elog(VERBOSE, "Found illegal file in TABLESPACE_VERSION_DIRECTORY, filepath %s", relative);
}
}
else if (sscanf_result == 1)
{
Assert(S_ISDIR(file->mode));
elog(VERBOSE, "dboid %u, filepath %s", file->dbOid, relative);
file->is_database = true;
}
else if (sscanf_result == 2)
{
elog(VERBOSE, "dboid %u, filename %s, filepath %s", file->dbOid, filename, relative);
}
else
{
elog(VERBOSE, "Illegal file filepath %s", relative);
}
}
}
else
{
elog(VERBOSE,"other dir or file, filepath %s", relative);
}
if (strcmp(filename, "ptrack_init") == 0)
{
/* Do not backup ptrack_init files */
pgFileFree(file);
parray_remove(files, i);
i--;
continue;
}
/* Check files located inside database directories including directory 'global' */
if (filename[0] != '\0' && file->tblspcOid != 0)
{
if (strcmp(filename, "pg_internal.init") == 0)
{
/* Do not pg_internal.init files
* (they contain some cache entries, so it's fine) */
pgFileFree(file);
parray_remove(files, i);
i--;
continue;
}
else if (filename[0] == 't' && isdigit(filename[1]))
{
elog(VERBOSE, "temp file, filepath %s", relative);
/* Do not backup temp files */
pgFileFree(file);
parray_remove(files, i);
i--;
continue;
}
else if (isdigit(filename[0]))
if (strcmp(file->forkName, "init") == 0)
{
/*
* TODO TODO TODO Files of this type can be compressed by cfs.
* Check that and do not mark them with 'is_datafile' flag.
* Do not backup files of unlogged relations.
* scan filelist backward and exclude these files.
*/
char *forkNameptr;
char suffix[MAXPGPATH];
int unlogged_file_num = i - 1;
pgFile *unlogged_file = (pgFile *) parray_get(files,
unlogged_file_num);
forkNameptr = strstr(filename, "_");
if (forkNameptr != NULL)
{
/* auxiliary fork of the relfile */
sscanf(filename, "%u_%s", &(file->relOid), file->forkName);
elog(VERBOSE, "relOid %u, forkName %s, filepath %s", file->relOid, file->forkName, relative);
unlogged_file_reloid = file->relOid;
/* handle unlogged relations */
if (strcmp(file->forkName, "init") == 0)
{
/*
* Do not backup files of unlogged relations.
* scan filelist backward and exclude these files.
*/
int unlogged_file_num = i-1;
pgFile *unlogged_file = (pgFile *) parray_get(files, unlogged_file_num);
while (unlogged_file_num >= 0 &&
(unlogged_file_reloid != 0) &&
(unlogged_file->relOid == unlogged_file_reloid))
{
pgFileFree(unlogged_file);
parray_remove(files, unlogged_file_num);
unlogged_file_reloid = file->relOid;
while (unlogged_file_num >= 0 &&
(unlogged_file_reloid != 0) &&
(unlogged_file->relOid == unlogged_file_reloid))
{
unlogged_file->size = 0;
pgFileFree(unlogged_file);
parray_remove(files, unlogged_file_num);
unlogged_file_num--;
i--;
unlogged_file = (pgFile *) parray_get(files, unlogged_file_num);
}
}
else if (strcmp(file->forkName, "ptrack") == 0)
{
/* Do not backup ptrack files */
pgFileFree(file);
parray_remove(files, i);
i--;
}
else if ((unlogged_file_reloid != 0) &&
(file->relOid == unlogged_file_reloid))
{
/* Do not backup forks of unlogged relations */
pgFileFree(file);
parray_remove(files, i);
i--;
}
continue;
}
sscanf_result = sscanf(filename, "%u.%d.%s", &(file->relOid), &(file->segno), suffix);
if (sscanf_result == 0)
{
elog(ERROR, "cannot parse filepath %s", relative);
}
else if (sscanf_result == 1)
{
/* first segment of the relfile */
elog(VERBOSE, "relOid %u, segno %d, filepath %s", file->relOid, 0, relative);
if (strcmp(relative + strlen(relative) - strlen("cfm"), "cfm") == 0)
{
/* reloid.cfm */
elog(VERBOSE, "Found cfm file %s", relative);
}
else
{
elog(VERBOSE, "Found first segment of the relfile %s", relative);
file->is_datafile = true;
}
}
else if (sscanf_result == 2)
{
/* not first segment of the relfile */
elog(VERBOSE, "relOid %u, segno %d, filepath %s", file->relOid, file->segno, relative);
file->is_datafile = true;
}
else
{
/*
* some cfs specific file.
* It is not datafile, because we cannot read it block by block
*/
elog(VERBOSE, "relOid %u, segno %d, suffix %s, filepath %s", file->relOid, file->segno, suffix, relative);
}
if ((unlogged_file_reloid != 0) &&
(file->relOid == unlogged_file_reloid))
{
/* Do not backup segments of unlogged files */
pgFileFree(file);
parray_remove(files, i);
unlogged_file_num--;
i--;
unlogged_file = (pgFile *) parray_get(files,
unlogged_file_num);
}
}
}
i++;
}
}
@@ -2617,6 +2438,10 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
static uint32 prevtimeline = 0;
static XLogRecPtr prevpos = InvalidXLogRecPtr;
/* check for interrupt */
if (interrupted)
elog(ERROR, "Interrupted during backup");
/* we assume that we get called once at the end of each segment */
if (segment_finished)
elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"),
src/dir.c
+285 -178
View File
@@ -17,6 +17,8 @@
#include <dirent.h>
#include <time.h>
#include "catalog/catalog.h"
#include "catalog/pg_tablespace.h"
#include "datapagemap.h"
/*
@@ -88,9 +90,10 @@ static char *pgdata_exclude_files_non_exclusive[] =
static int BlackListCompare(const void *str1, const void *str2);
static bool dir_check_file(const char *root, pgFile *file);
static void dir_list_file_internal(parray *files, const char *root,
bool exclude, bool omit_symlink,
bool add_root, parray *black_list);
pgFile *parent, bool exclude,
bool omit_symlink, parray *black_list);
/*
* Create directory, also create parent directories if necessary.
@@ -145,7 +148,9 @@ pgFileNew(const char *path, bool omit_symlink)
pgFile *
pgFileInit(const char *path)
{
pgFile *file;
pgFile *file;
char *file_name;
file = (pgFile *) pgut_malloc(sizeof(pgFile));
file->size = 0;
@@ -163,8 +168,21 @@ pgFileInit(const char *path)
file->segno = 0;
file->is_database = false;
file->forkName = pgut_malloc(MAXPGPATH);
file->forkName[0] = '\0';
file->path = pgut_malloc(strlen(path) + 1);
strcpy(file->path, path); /* enough buffer size guaranteed */
/* Get file name from the path */
file_name = strrchr(file->path, '/');
if (file_name == NULL)
file->name = file->path;
else
{
file_name++;
file->name = file_name;
}
file->is_cfs = false;
file->exists_in_prev = false; /* can change only in Incremental backup. */
file->n_blocks = -1; /* can change only in DELTA backup. Number of blocks readed during backup */
@@ -319,6 +337,7 @@ void
dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
bool add_root)
{
pgFile *file;
parray *black_list = NULL;
char path[MAXPGPATH];
@@ -353,213 +372,301 @@ dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
parray_qsort(black_list, BlackListCompare);
}
dir_list_file_internal(files, root, exclude, omit_symlink, add_root,
black_list);
file = pgFileNew(root, false);
if (file == NULL)
return;
if (!S_ISDIR(file->mode))
{
elog(WARNING, "Skip \"%s\": unexpected file format", file->path);
return;
}
if (add_root)
parray_append(files, file);
dir_list_file_internal(files, root, file, exclude, omit_symlink, black_list);
parray_qsort(files, pgFileComparePath);
}
/*
* TODO Add comment
* Check file or directory.
*
* Check for exclude.
* Extract information about the file parsing its name.
* Skip files:
* - skip temp tables files
* - skip unlogged tables files
* Set flags for:
* - database directories
* - datafiles
*/
static void
dir_list_file_internal(parray *files, const char *root, bool exclude,
bool omit_symlink, bool add_root, parray *black_list)
static bool
dir_check_file(const char *root, pgFile *file)
{
pgFile *file;
const char *rel_path;
int i;
int sscanf_res;
file = pgFileNew(root, omit_symlink);
if (file == NULL)
return;
/* skip if the file is in black_list defined by user */
if (black_list && parray_bsearch(black_list, root, BlackListCompare))
/* Check if we need to exclude file by name */
if (S_ISREG(file->mode))
{
elog(LOG, "Skip file \"%s\": file is in the user's black list", file->path);
return;
}
/*
* Add to files list only files, links and directories. Skip sockets and
* other unexpected file formats.
*/
if (!S_ISDIR(file->mode) && !S_ISLNK(file->mode) && !S_ISREG(file->mode))
{
elog(WARNING, "Skip file \"%s\": unexpected file format", file->path);
return;
}
if (add_root)
{
/* Skip files */
/* TODO Consider moving this check to parse_backup_filelist_filenames */
if (!S_ISDIR(file->mode) && exclude)
if (!exclusive_backup)
{
char *file_name;
int i;
/* Extract file name */
file_name = strrchr(file->path, '/');
if (file_name == NULL)
file_name = file->path;
else
{
file_name++;
file->name = file_name;
}
/* exclude backup_label and tablespace_map in non-exclusive backup */
if (!exclusive_backup)
{
for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++)
if (strcmp(file->name, pgdata_exclude_files_non_exclusive[i]) == 0)
{
/* Skip */
elog(VERBOSE, "Excluding file: %s", file->name);
return;
}
}
/* Check if we need to exclude file by name */
for (i = 0; pgdata_exclude_files[i]; i++)
if (strcmp(file->name, pgdata_exclude_files[i]) == 0)
for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++)
if (strcmp(file->name,
pgdata_exclude_files_non_exclusive[i]) == 0)
{
/* Skip */
elog(VERBOSE, "Excluding file: %s", file->name);
return;
return false;
}
}
parray_append(files, file);
}
/* chase symbolic link chain and find regular file or directory */
while (S_ISLNK(file->mode))
for (i = 0; pgdata_exclude_files[i]; i++)
if (strcmp(file->name, pgdata_exclude_files[i]) == 0)
{
/* Skip */
elog(VERBOSE, "Excluding file: %s", file->name);
return false;
}
}
/*
* If the directory name is in the exclude list, do not list the
* contents.
*/
else if (S_ISDIR(file->mode))
{
ssize_t len;
char linked[MAXPGPATH];
len = readlink(file->path, linked, sizeof(linked));
if (len < 0)
elog(ERROR, "cannot read link \"%s\": %s", file->path,
strerror(errno));
if (len >= sizeof(linked))
elog(ERROR, "symbolic link \"%s\" target is too long\n",
file->path);
linked[len] = '\0';
file->linked = pgut_strdup(linked);
/* make absolute path to read linked file */
if (linked[0] != '/')
/*
* If the item in the exclude list starts with '/', compare to
* the absolute path of the directory. Otherwise compare to the
* directory name portion.
*/
for (i = 0; pgdata_exclude_dir[i]; i++)
{
char dname[MAXPGPATH];
char absolute[MAXPGPATH];
strncpy(dname, file->path, lengthof(dname));
join_path_components(absolute, dname, linked);
file = pgFileNew(absolute, omit_symlink);
/* Full-path exclude*/
if (pgdata_exclude_dir[i][0] == '/')
{
if (strcmp(file->path, pgdata_exclude_dir[i]) == 0)
{
elog(VERBOSE, "Excluding directory content: %s",
file->name);
return false;
}
}
else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
{
elog(VERBOSE, "Excluding directory content: %s",
file->name);
return false;
}
}
else
file = pgFileNew(file->linked, omit_symlink);
/* linked file is not found, stop following link chain */
if (file == NULL)
return;
parray_append(files, file);
}
rel_path = GetRelativePath(file->path, root);
/*
* If the entry was a directory, add it to the list and add call this
* function recursivelly.
* If the directory name is in the exclude list, do not list the contents.
* Do not copy tablespaces twice. It may happen if the tablespace is located
* inside the PGDATA.
*/
while (S_ISDIR(file->mode))
if (S_ISDIR(file->mode) &&
strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0)
{
bool skip = false;
DIR *dir;
struct dirent *dent;
Oid tblspcOid;
char tmp_rel_path[MAXPGPATH];
if (exclude)
/*
* Valid path for the tablespace is
* pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY
*/
if (!path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path))
return false;
sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%s",
&tblspcOid, tmp_rel_path);
if (sscanf_res == 0)
return false;
}
if (path_is_prefix_of_path("global", rel_path))
{
file->tblspcOid = GLOBALTABLESPACE_OID;
if (S_ISDIR(file->mode) && strcmp(file->name, "global") == 0)
file->is_database = true;
}
else if (path_is_prefix_of_path("base", rel_path))
{
file->tblspcOid = DEFAULTTABLESPACE_OID;
sscanf(rel_path, "base/%u/", &(file->dbOid));
if (S_ISDIR(file->mode) && strcmp(file->name, "base") != 0)
file->is_database = true;
}
else if (path_is_prefix_of_path(PG_TBLSPC_DIR, rel_path))
{
char tmp_rel_path[MAXPGPATH];
sscanf_res = sscanf(rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/",
&(file->tblspcOid), tmp_rel_path,
&(file->dbOid));
if (sscanf_res == 3 && S_ISDIR(file->mode) &&
strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) == 0)
file->is_database = true;
}
/* Do not backup ptrack_init files */
if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0)
return false;
/*
* Check files located inside database directories including directory
* 'global'
*/
if (S_ISREG(file->mode) && file->tblspcOid != 0 &&
file->name && file->name[0])
{
if (strcmp(file->name, "pg_internal.init") == 0)
return false;
/* Do not backup temp files */
else if (file->name[0] == 't' && isdigit(file->name[1]))
return false;
else if (isdigit(file->name[0]))
{
int i;
char *dirname;
char *fork_name;
int len;
char suffix[MAXPGPATH];
/* skip entry which matches exclude list */
dirname = strrchr(file->path, '/');
if (dirname == NULL)
dirname = file->path;
fork_name = strstr(file->name, "_");
if (fork_name)
{
/* Auxiliary fork of the relfile */
sscanf(file->name, "%u_%s", &(file->relOid), file->forkName);
/* Do not backup ptrack files */
if (strcmp(file->forkName, "ptrack") == 0)
return false;
}
else
{
dirname++;
file->name = dirname;
}
len = strlen(file->name);
/* reloid.cfm */
if (len > 3 && strcmp(file->name + len - 3, "cfm") == 0)
return true;
/*
* If the item in the exclude list starts with '/', compare to the
* absolute path of the directory. Otherwise compare to the directory
* name portion.
*/
for (i = 0; exclude && pgdata_exclude_dir[i]; i++)
{
/* Full-path exclude*/
if (pgdata_exclude_dir[i][0] == '/')
{
if (strcmp(file->path, pgdata_exclude_dir[i]) == 0)
{
skip = true;
break;
}
}
else if (strcmp(file->name, pgdata_exclude_dir[i]) == 0)
{
skip = true;
break;
}
}
if (skip)
{
elog(VERBOSE, "Excluding directory content: %s", file->name);
break;
sscanf_res = sscanf(file->name, "%u.%d.%s", &(file->relOid),
&(file->segno), suffix);
if (sscanf_res == 0)
elog(ERROR, "Cannot parse file name \"%s\"", file->name);
else if (sscanf_res == 1 || sscanf_res == 2)
file->is_datafile = true;
}
}
/* open directory and list contents */
dir = opendir(file->path);
if (dir == NULL)
{
if (errno == ENOENT)
{
/* maybe the direcotry was removed */
return;
}
elog(ERROR, "cannot open directory \"%s\": %s",
file->path, strerror(errno));
}
errno = 0;
while ((dent = readdir(dir)))
{
char child[MAXPGPATH];
/* skip entries point current dir or parent dir */
if (strcmp(dent->d_name, ".") == 0 ||
strcmp(dent->d_name, "..") == 0)
continue;
join_path_components(child, file->path, dent->d_name);
dir_list_file_internal(files, child, exclude, omit_symlink, true, black_list);
}
if (errno && errno != ENOENT)
{
int errno_tmp = errno;
closedir(dir);
elog(ERROR, "cannot read directory \"%s\": %s",
file->path, strerror(errno_tmp));
}
closedir(dir);
break; /* pseudo loop */
}
return true;
}
/*
* List files in "root" directory. If "exclude" is true do not add into "files"
* files from pgdata_exclude_files and directories from pgdata_exclude_dir.
*/
static void
dir_list_file_internal(parray *files, const char *root, pgFile *parent,
bool exclude, bool omit_symlink, parray *black_list)
{
DIR *dir;
struct dirent *dent;
if (!S_ISDIR(parent->mode))
elog(ERROR, "\"%s\" is not a directory", parent->path);
/* Open directory and list contents */
dir = opendir(parent->path);
if (dir == NULL)
{
if (errno == ENOENT)
{
/* Maybe the directory was removed */
return;
}
elog(ERROR, "cannot open directory \"%s\": %s",
parent->path, strerror(errno));
}
errno = 0;
while ((dent = readdir(dir)))
{
pgFile *file;
char child[MAXPGPATH];
join_path_components(child, parent->path, dent->d_name);
file = pgFileNew(child, omit_symlink);
if (file == NULL)
continue;
/* Skip entries point current dir or parent dir */
if (S_ISDIR(file->mode) &&
(strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0))
{
pgFileFree(file);
continue;
}
/*
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (!S_ISDIR(file->mode) && !S_ISREG(file->mode))
{
elog(WARNING, "Skip \"%s\": unexpected file format", file->path);
pgFileFree(file);
continue;
}
/* Skip if the directory is in black_list defined by user */
if (black_list && parray_bsearch(black_list, file->path,
BlackListCompare))
{
elog(LOG, "Skip \"%s\": it is in the user's black list", file->path);
pgFileFree(file);
continue;
}
/* We add the directory anyway */
if (S_ISDIR(file->mode))
parray_append(files, file);
if (exclude && !dir_check_file(root, file))
{
if (S_ISREG(file->mode))
pgFileFree(file);
/* Skip */
continue;
}
/* At least add the file */
if (S_ISREG(file->mode))
parray_append(files, file);
/*
* If the entry is a directory call dir_list_file_internal()
* recursively.
*/
if (S_ISDIR(file->mode))
dir_list_file_internal(files, root, file, exclude, omit_symlink,
black_list);
}
if (errno && errno != ENOENT)
{
int errno_tmp = errno;
closedir(dir);
elog(ERROR, "cannot read directory \"%s\": %s",
parent->path, strerror(errno_tmp));
}
closedir(dir);
}
/*
src/utils/logger.c
+11 -11
View File
@@ -131,6 +131,12 @@ exit_if_necessary(int elevel)
/* Interrupt other possible routines */
interrupted = true;
if (loggin_in_progress)
{
loggin_in_progress = false;
pthread_mutex_unlock(&log_file_mutex);
}
/* If this is not the main thread then don't call exit() */
#ifdef WIN32
if (main_tid != GetCurrentThreadId())
@@ -168,11 +174,8 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args)
/*
* There is no need to lock if this is elog() from upper elog().
*/
if (!loggin_in_progress)
{
pthread_mutex_lock(&log_file_mutex);
loggin_in_progress = true;
}
pthread_mutex_lock(&log_file_mutex);
loggin_in_progress = true;
/* We need copy args only if we need write to error log file */
if (write_to_error_log)
@@ -250,13 +253,10 @@ elog_internal(int elevel, bool file_only, const char *fmt, va_list args)
va_end(std_args);
}
if (loggin_in_progress)
{
pthread_mutex_unlock(&log_file_mutex);
loggin_in_progress = false;
}
exit_if_necessary(elevel);
loggin_in_progress = false;
pthread_mutex_unlock(&log_file_mutex);
}
/*
tests/backup_test.py
+53 -6
View File
@@ -422,7 +422,6 @@ class BackupTest(ProbackupTest, unittest.TestCase):
self.del_test_dir(module_name, fname)
# @unittest.skip("skip")
@unittest.expectedFailure
def test_tablespace_in_pgdata_pgpro_1376(self):
"""PGPRO-1376 """
fname = self.id().split('.')[3]
@@ -443,20 +442,58 @@ class BackupTest(ProbackupTest, unittest.TestCase):
tblspc_path=(os.path.join(node.data_dir, 'pg_clog', '100500'))
)
self.create_tblspace_in_node(
node, 'tblspace2',
tblspc_path=(os.path.join(node.data_dir))
)
node.safe_psql(
"postgres",
"create table t_heap tablespace tblspace1 as select 1 as id, "
"create table t_heap1 tablespace tblspace1 as select 1 as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1000) i")
node.safe_psql(
"postgres",
"create table t_heap2 tablespace tblspace2 as select 1 as id, "
"md5(i::text) as text, "
"md5(repeat(i::text,10))::tsvector as tsvector "
"from generate_series(0,1000) i")
try:
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
# we should die here because exception is what we expect to happen
self.assertEqual(
1, 0,
"Expecting Error because of too many levels "
"of symbolic linking\n"
" Output: {0} \n CMD: {1}".format(
repr(self.output), self.cmd))
except ProbackupException as e:
self.assertTrue(
'Too many levels of symbolic links' in e.message,
"\n Unexpected Error Message: {0}\n CMD: {1}".format(
repr(e.message), self.cmd))
node.safe_psql(
"postgres",
"drop table t_heap2")
node.safe_psql(
"postgres",
"drop tablespace tblspace2")
self.backup_node(
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
backup_dir, 'node', node, backup_type="full",
options=["-j", "4", "--stream"])
pgdata = self.pgdata_content(node.data_dir)
relfilenode = node.safe_psql(
"postgres",
"select 't_heap'::regclass::oid"
"select 't_heap1'::regclass::oid"
).rstrip()
list = []
@@ -466,7 +503,8 @@ class BackupTest(ProbackupTest, unittest.TestCase):
path = os.path.join(root, file)
list = list + [path]
if len(list) > 0:
# We expect that relfilenode occures only once
if len(list) > 1:
message = ""
for string in list:
message = message + string + "\n"
@@ -475,3 +513,12 @@ class BackupTest(ProbackupTest, unittest.TestCase):
"Following file copied twice by backup:\n {0}".format(
message)
)
node.cleanup()
self.restore_node(
backup_dir, 'node', node, options=["-j", "4"])
if self.paranoia:
pgdata_restored = self.pgdata_content(node.data_dir)
self.compare_pgdata(pgdata, pgdata_restored)
tests/cfs_restore.py
+1 -1
View File
@@ -74,7 +74,7 @@ class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase):
"""
Case: Restore empty tablespace from valid full backup.
"""
self.node.stop({"-m": "immediate"})
self.node.stop(["-m", "immediate"])
self.node.cleanup()
shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name))
tests/helpers/ptrack_helpers.py
+3 -1
View File
@@ -259,7 +259,9 @@ class ProbackupTest(object):
tblspc_name, tblspc_path)
if cfs:
cmd += " with (compression=true)"
os.makedirs(tblspc_path)
if not os.path.exists(tblspc_path):
os.makedirs(tblspc_path)
res = node.safe_psql("postgres", cmd)
# Check that tablespace was successfully created
# self.assertEqual(