db/pg_probackup
db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2025-01-05 13:20:31 +02:00
Code Issues Releases Activity

Remove sanityChecks(). Fix add_files()

Browse Source
This commit is contained in:
Artur Zakirov 2017-02-16 17:23:43 +03:00
parent 492a85606f
commit a997235f89
6 changed files with 132 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

157
backup.c
View File

@ -126,9 +126,6 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
/* Initialize size summary */
current.data_bytes = 0;
/* do some checks on the node */
sanityChecks();
/*
* Obtain current timeline by scanning control file, theh LSN
* obtained at output of pg_start_backup or pg_stop_backup does
@ -152,11 +149,9 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
pgBackup *prev_backup;
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
if (prev_backup == NULL)
elog(ERROR, "Timeline has changed since last full backup."
elog(ERROR, "Timeline has changed since last full backup. "
"Create new full backup before an incremental one.");
}
@ -233,8 +228,7 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint)
if (current.backup_mode == BACKUP_MODE_DIFF_PAGE ||
current.backup_mode == BACKUP_MODE_DIFF_PTRACK)
{
/* find last completed database backup */
prev_backup = catalog_get_last_data_backup(backup_list, current.tli);
Assert(prev_backup);
pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt),
DATABASE_FILE_LIST);
prev_files = dir_read_file_list(pgdata, prev_file_txt);
@ -1337,8 +1331,8 @@ backup_files(void *arg)
static void
add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
{
parray *list_file;
int i;
parray *list_file;
size_t i;
list_file = parray_new();
@ -1346,12 +1340,12 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
dir_list_file(list_file, root, true, true, add_root);
/* mark files that are possible datafile as 'datafile' */
for (i = 0; i < (int) parray_num(list_file); i++)
for (i = 0; i < parray_num(list_file); i++)
{
pgFile *file = (pgFile *) parray_get(list_file, i);
char *relative;
char *fname;
int path_len;
pgFile *file = (pgFile *) parray_get(list_file, i);
char *relative;
char *fname;
size_t path_len;
/* data file must be a regular file */
if (!S_ISREG(file->mode))
@ -1367,6 +1361,10 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
/* Get file name from path */
fname = last_dir_separator(relative);
if (fname == NULL)
fname = relative;
else
fname++;
/* Remove temp tables from the list */
if (fname[0] == 't' && isdigit(fname[1]))
@ -1379,32 +1377,41 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
path_len = strlen(file->path);
/* Get link ptrack file to relations files */
if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0)
if (path_len > 6 &&
strncmp(file->path + (path_len - 6), "ptrack", 6) == 0)
{
pgFile *search_file;
pgFile **pre_search_file;
int segno = 0;
while(true) {
pgFile tmp_file;
pgFile *search_file;
pgFile **pre_search_file;
int segno = 0;
while (true)
{
pgFile tmp_file;
tmp_file.path = pg_strdup(file->path);
/* Segno fits into 6 digits since it is not more than 4000 */
if (segno > 0)
sprintf(tmp_file.path+path_len-7, ".%d", segno);
sprintf(tmp_file.path + path_len - 7, ".%d", segno);
else
tmp_file.path[path_len-7] = '\0';
tmp_file.path[path_len - 7] = '\0';
pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath);
pre_search_file = (pgFile **) parray_bsearch(list_file,
&tmp_file,
pgFileComparePath);
if (pre_search_file != NULL)
{
search_file = *pre_search_file;
search_file->ptrack_path = pg_strdup(file->path);
search_file->segno = segno;
} else {
}
else
{
pg_free(tmp_file.path);
break;
}
pg_free(tmp_file.path);
segno++;
}
@ -1413,63 +1420,24 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
pgFileFree(file);
parray_remove(list_file, i);
i--;
continue;
}
/* compress map file it is not data file */
if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0)
continue;
/* name of data file start with digit */
if (fname == NULL)
fname = relative;
else
fname++;
if (!isdigit(fname[0]))
continue;
file->is_datafile = true;
else if (path_len > 4 &&
strncmp(file->path + (path_len - 4), ".cfm", 4) == 0)
{
int find_dot;
int check_digit;
char *text_segno;
for(find_dot = path_len-1; file->path[find_dot] != '.' && find_dot >= 0; find_dot--);
if (find_dot <= 0)
continue;
pgFile **pre_search_file;
pgFile tmp_file;
text_segno = file->path + find_dot + 1;
for(check_digit=0; text_segno[check_digit] != '\0'; check_digit++)
if (!isdigit(text_segno[check_digit]))
{
check_digit = -1;
break;
}
if (check_digit == -1)
continue;
file->segno = (int) strtol(text_segno, NULL, 10);
}
}
/* mark cfs relations as not data */
for (i = 0; i < (int) parray_num(list_file); i++)
{
pgFile *file = (pgFile *) parray_get(list_file, i);
int path_len = (int) strlen(file->path);
if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0)
{
pgFile **pre_search_file;
pgFile tmp_file;
tmp_file.path = pg_strdup(file->path);
tmp_file.path[path_len-4] = '\0';
tmp_file.path[path_len - 4] = '\0';
pre_search_file = (pgFile **) parray_bsearch(list_file,
&tmp_file, pgFileComparePath);
&tmp_file,
pgFileComparePath);
if (pre_search_file != NULL)
{
FileMap* map;
int md = open(file->path, O_RDWR|PG_BINARY, 0);
FileMap *map;
int md = open(file->path, O_RDWR|PG_BINARY, 0);
if (md < 0)
elog(ERROR, "add_files(). cannot open cfm file '%s'", file->path);
@ -1485,16 +1453,51 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata)
(*pre_search_file)->is_datafile = false;
if (cfs_munmap(map) < 0)
elog(LOG, "add_files(). CFS failed to unmap file %s: %m", file->path);
elog(LOG, "add_files(). CFS failed to unmap file %s: %m",
file->path);
if (close(md) < 0)
elog(LOG, "add_files(). CFS failed to close file %s: %m", file->path);
elog(LOG, "add_files(). CFS failed to close file %s: %m",
file->path);
}
else
elog(ERROR, "corresponding segment '%s' is not found", tmp_file.path);
elog(ERROR, "corresponding segment '%s' is not found",
tmp_file.path);
pg_free(tmp_file.path);
}
/* name of data file start with digit */
else if (isdigit(fname[0]))
{
int find_dot;
int check_digit;
char *text_segno;
file->is_datafile = true;
/*
* Find segment number.
*/
for (find_dot = (int) path_len - 1;
file->path[find_dot] != '.' && find_dot >= 0;
find_dot--);
/* There is not segment number */
if (find_dot <= 0)
continue;
text_segno = file->path + find_dot + 1;
for (check_digit = 0; text_segno[check_digit] != '\0'; check_digit++)
if (!isdigit(text_segno[check_digit]))
{
check_digit = -1;
break;
}
if (check_digit != -1)
file->segno = (int) strtol(text_segno, NULL, 10);
}
}
parray_concat(files, list_file);
}

2
delete.c
View File

@ -283,7 +283,7 @@ pgBackupDeleteFiles(pgBackup *backup)
parray_qsort(files, pgFileComparePathDesc);
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
pgFile *file = (pgFile *) parray_get(files, i);
/* print progress */
elog(LOG, "delete file(%zd/%lu) \"%s\"", i + 1,

59
dir.c
View File

@ -249,9 +249,6 @@ BlackListCompare(const void *str1, const void *str2)
* List files, symbolic links and directories in the directory "root" and add
* pgFile objects to "files". We add "root" to "files" if add_root is true.
*
* If the sub-directory name is in "exclude" list, the sub-directory itself is
* listed but the contents of the sub-directory is ignored.
*
* When omit_symlink is true, symbolic link is ignored and only file or
* directory llnked to will be listed.
*/
@ -259,40 +256,43 @@ void
dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink,
bool add_root)
{
char path[MAXPGPATH];
char buf[MAXPGPATH * 2];
char black_item[MAXPGPATH * 2];
parray *black_list = NULL;
parray *black_list = NULL;
char path[MAXPGPATH];
join_path_components(path, backup_path, PG_BLACK_LIST);
if (root && pgdata && strcmp(root, pgdata) == 0 &&
fileExists(path))
/* List files with black list */
if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path))
{
FILE *black_list_file = NULL;
FILE *black_list_file = NULL;
char buf[MAXPGPATH * 2];
char black_item[MAXPGPATH * 2];
black_list = parray_new();
black_list_file = fopen(path, "r");
if (black_list_file == NULL)
elog(ERROR, "cannot open black_list: %s",
strerror(errno));
elog(ERROR, "cannot open black_list: %s", strerror(errno));
while (fgets(buf, lengthof(buf), black_list_file) != NULL)
{
join_path_components(black_item, pgdata, buf);
if (black_item[strlen(black_item) - 1] == '\n')
black_item[strlen(black_item) - 1] = '\0';
if (black_item[0] == '#' || black_item[0] == '\0')
continue;
parray_append(black_list, black_item);
}
fclose(black_list_file);
parray_qsort(black_list, BlackListCompare);
dir_list_file_internal(files, root, exclude, omit_symlink, add_root, black_list);
parray_qsort(files, pgFileComparePath);
}
else
{
dir_list_file_internal(files, root, exclude, omit_symlink, add_root, NULL);
parray_qsort(files, pgFileComparePath);
}
dir_list_file_internal(files, root, exclude, omit_symlink, add_root,
black_list);
parray_qsort(files, pgFileComparePath);
}
void
@ -561,7 +561,8 @@ dir_print_file_list(FILE *out, const parray *files, const char *root, const char
fprintf(out, " %s", timestamp);
}
fprintf(out, " %d %d\n", file->generation, file->is_partial_copy);
fprintf(out, " " UINT64_FORMAT " %d\n",
file->generation, file->is_partial_copy);
}
}
@ -681,30 +682,34 @@ dir_read_file_list(const char *root, const char *file_txt)
return files;
}
/* copy contents of directory from_root into to_root */
/*
* Copy contents of directory from_root into to_root.
*/
void
dir_copy_files(const char *from_root, const char *to_root)
{
int i;
parray *files = parray_new();
size_t i;
parray *files = parray_new();
/* don't copy root directory */
dir_list_file(files, from_root, false, true, false);
for (i = 0; i < parray_num(files); i++)
{
pgFile *file = (pgFile *) parray_get(files, i);
pgFile *file = (pgFile *) parray_get(files, i);
if (S_ISDIR(file->mode))
{
char to_path[MAXPGPATH];
join_path_components(to_path, to_root, file->path + strlen(from_root) + 1);
char to_path[MAXPGPATH];
join_path_components(to_path, to_root,
file->path + strlen(from_root) + 1);
if (verbose && !check)
elog(LOG, "creating directory \"%s\"",
file->path + strlen(from_root) + 1);
if (!check)
dir_create_dir(to_path, DIR_PERMISSION);
continue;
}
else if (S_ISREG(file->mode))
{

6
pg_probackup.h
View File

@ -70,10 +70,10 @@ typedef struct pgFile
char *path; /* path of the file */
char *ptrack_path;
int segno; /* Segment number for ptrack */
int generation; /* Generation of compressed file.
uint64 generation; /* Generation of compressed file.
* -1 for non-compressed files */
int is_partial_copy; /* for compressed files.
* 1 if backed up via copy_file_partly() */
int is_partial_copy; /* for compressed files.
* 1 if backed up via copy_file_partly() */
volatile uint32 lock;
datapagemap_t pagemap;
} pgFile;

17
tests/retention_test.py
View File

@ -63,18 +63,29 @@ class RetentionTest(ProbackupTest, unittest.TestCase):
self.init_pb(node)
with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf:
conf.write("REDUNDANCY=1\n")
conf.write("WINDOW=2\n")
conf.write("WINDOW=1\n")
# All backups will be keeped
# Make backups to be purged
self.backup_pb(node)
self.backup_pb(node, backup_type="page")
# Make backup to be keeped
self.backup_pb(node)
backups = path.join(self.backup_dir(node), "backups")
days_delta = 5
for backup in listdir(backups):
with open(path.join(backups, backup, "backup.conf"), "a") as conf:
conf.write("RECOVERY_TIME='{:%Y-%m-%d %H:%M:%S}'\n".format(
datetime.now() - timedelta(days=days_delta)))
days_delta -= 1
# Make backup to be keeped
self.backup_pb(node, backup_type="page")
self.assertEqual(len(self.show_pb(node)), 4)
# Purge backups
self.retention_purge_pb(node)
self.assertEqual(len(self.show_pb(node)), 4)
self.assertEqual(len(self.show_pb(node)), 2)
node.stop()

17
util.c
View File

@ -72,20 +72,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
checkControlFile(ControlFile);
}
void
sanityChecks(void)
{
ControlFileData ControlFile;
char *buffer;
size_t size;
/* First fetch file... */
buffer = slurpFile(pgdata, "global/pg_control", &size, false);
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);
}
XLogRecPtr
get_last_ptrack_lsn(void)
{
@ -114,8 +100,9 @@ get_current_timeline(bool safe)
/* First fetch file... */
buffer = slurpFile(pgdata, "global/pg_control", &size, safe);
if (buffer == NULL)
if (safe && buffer == NULL)
return 0;
digestControlFile(&ControlFile, buffer, size);
pg_free(buffer);