From a997235f89bfbf9341a4eb414b351ec7066a0337 Mon Sep 17 00:00:00 2001 From: Artur Zakirov Date: Thu, 16 Feb 2017 17:23:43 +0300 Subject: [PATCH] Remove sanityChecks(). Fix add_files() --- backup.c | 157 ++++++++++++++++++++-------------------- delete.c | 2 +- dir.c | 59 ++++++++------- pg_probackup.h | 6 +- tests/retention_test.py | 17 ++++- util.c | 17 +---- 6 files changed, 132 insertions(+), 126 deletions(-) diff --git a/backup.c b/backup.c index e57c6d3b..7f6b50b3 100644 --- a/backup.c +++ b/backup.c @@ -126,9 +126,6 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint) /* Initialize size summary */ current.data_bytes = 0; - /* do some checks on the node */ - sanityChecks(); - /* * Obtain current timeline by scanning control file, theh LSN * obtained at output of pg_start_backup or pg_stop_backup does @@ -152,11 +149,9 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint) if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - pgBackup *prev_backup; - prev_backup = catalog_get_last_data_backup(backup_list, current.tli); if (prev_backup == NULL) - elog(ERROR, "Timeline has changed since last full backup." + elog(ERROR, "Timeline has changed since last full backup. " "Create new full backup before an incremental one."); } @@ -233,8 +228,7 @@ do_backup_database(parray *backup_list, bool smooth_checkpoint) if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || current.backup_mode == BACKUP_MODE_DIFF_PTRACK) { - /* find last completed database backup */ - prev_backup = catalog_get_last_data_backup(backup_list, current.tli); + Assert(prev_backup); pgBackupGetPath(prev_backup, prev_file_txt, lengthof(prev_file_txt), DATABASE_FILE_LIST); prev_files = dir_read_file_list(pgdata, prev_file_txt); @@ -1337,8 +1331,8 @@ backup_files(void *arg) static void add_files(parray *files, const char *root, bool add_root, bool is_pgdata) { - parray *list_file; - int i; + parray *list_file; + size_t i; list_file = parray_new(); @@ -1346,12 +1340,12 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata) dir_list_file(list_file, root, true, true, add_root); /* mark files that are possible datafile as 'datafile' */ - for (i = 0; i < (int) parray_num(list_file); i++) + for (i = 0; i < parray_num(list_file); i++) { - pgFile *file = (pgFile *) parray_get(list_file, i); - char *relative; - char *fname; - int path_len; + pgFile *file = (pgFile *) parray_get(list_file, i); + char *relative; + char *fname; + size_t path_len; /* data file must be a regular file */ if (!S_ISREG(file->mode)) @@ -1367,6 +1361,10 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata) /* Get file name from path */ fname = last_dir_separator(relative); + if (fname == NULL) + fname = relative; + else + fname++; /* Remove temp tables from the list */ if (fname[0] == 't' && isdigit(fname[1])) @@ -1379,32 +1377,41 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata) path_len = strlen(file->path); /* Get link ptrack file to relations files */ - if (path_len > 6 && strncmp(file->path+(path_len-6), "ptrack", 6) == 0) + if (path_len > 6 && + strncmp(file->path + (path_len - 6), "ptrack", 6) == 0) { - pgFile *search_file; - pgFile **pre_search_file; - int segno = 0; - while(true) { - pgFile tmp_file; + pgFile *search_file; + pgFile **pre_search_file; + int segno = 0; + + while (true) + { + pgFile tmp_file; + tmp_file.path = pg_strdup(file->path); /* Segno fits into 6 digits since it is not more than 4000 */ if (segno > 0) - sprintf(tmp_file.path+path_len-7, ".%d", segno); + sprintf(tmp_file.path + path_len - 7, ".%d", segno); else - tmp_file.path[path_len-7] = '\0'; + tmp_file.path[path_len - 7] = '\0'; - pre_search_file = (pgFile **) parray_bsearch(list_file, &tmp_file, pgFileComparePath); + pre_search_file = (pgFile **) parray_bsearch(list_file, + &tmp_file, + pgFileComparePath); if (pre_search_file != NULL) { search_file = *pre_search_file; search_file->ptrack_path = pg_strdup(file->path); search_file->segno = segno; - } else { + } + else + { pg_free(tmp_file.path); break; } + pg_free(tmp_file.path); segno++; } @@ -1413,63 +1420,24 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata) pgFileFree(file); parray_remove(list_file, i); i--; - continue; } - /* compress map file it is not data file */ - if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0) - continue; - - /* name of data file start with digit */ - if (fname == NULL) - fname = relative; - else - fname++; - if (!isdigit(fname[0])) - continue; - - file->is_datafile = true; + else if (path_len > 4 && + strncmp(file->path + (path_len - 4), ".cfm", 4) == 0) { - int find_dot; - int check_digit; - char *text_segno; - for(find_dot = path_len-1; file->path[find_dot] != '.' && find_dot >= 0; find_dot--); - if (find_dot <= 0) - continue; + pgFile **pre_search_file; + pgFile tmp_file; - text_segno = file->path + find_dot + 1; - for(check_digit=0; text_segno[check_digit] != '\0'; check_digit++) - if (!isdigit(text_segno[check_digit])) - { - check_digit = -1; - break; - } - - if (check_digit == -1) - continue; - - file->segno = (int) strtol(text_segno, NULL, 10); - } - } - - /* mark cfs relations as not data */ - for (i = 0; i < (int) parray_num(list_file); i++) - { - pgFile *file = (pgFile *) parray_get(list_file, i); - int path_len = (int) strlen(file->path); - - if (path_len > 4 && strncmp(file->path+(path_len-4), ".cfm", 4) == 0) - { - pgFile **pre_search_file; - pgFile tmp_file; tmp_file.path = pg_strdup(file->path); - tmp_file.path[path_len-4] = '\0'; + tmp_file.path[path_len - 4] = '\0'; pre_search_file = (pgFile **) parray_bsearch(list_file, - &tmp_file, pgFileComparePath); + &tmp_file, + pgFileComparePath); if (pre_search_file != NULL) { - FileMap* map; - int md = open(file->path, O_RDWR|PG_BINARY, 0); + FileMap *map; + int md = open(file->path, O_RDWR|PG_BINARY, 0); + if (md < 0) elog(ERROR, "add_files(). cannot open cfm file '%s'", file->path); @@ -1485,16 +1453,51 @@ add_files(parray *files, const char *root, bool add_root, bool is_pgdata) (*pre_search_file)->is_datafile = false; if (cfs_munmap(map) < 0) - elog(LOG, "add_files(). CFS failed to unmap file %s: %m", file->path); + elog(LOG, "add_files(). CFS failed to unmap file %s: %m", + file->path); if (close(md) < 0) - elog(LOG, "add_files(). CFS failed to close file %s: %m", file->path); + elog(LOG, "add_files(). CFS failed to close file %s: %m", + file->path); } else - elog(ERROR, "corresponding segment '%s' is not found", tmp_file.path); + elog(ERROR, "corresponding segment '%s' is not found", + tmp_file.path); pg_free(tmp_file.path); } + /* name of data file start with digit */ + else if (isdigit(fname[0])) + { + int find_dot; + int check_digit; + char *text_segno; + + file->is_datafile = true; + + /* + * Find segment number. + */ + + for (find_dot = (int) path_len - 1; + file->path[find_dot] != '.' && find_dot >= 0; + find_dot--); + /* There is not segment number */ + if (find_dot <= 0) + continue; + + text_segno = file->path + find_dot + 1; + for (check_digit = 0; text_segno[check_digit] != '\0'; check_digit++) + if (!isdigit(text_segno[check_digit])) + { + check_digit = -1; + break; + } + + if (check_digit != -1) + file->segno = (int) strtol(text_segno, NULL, 10); + } } + parray_concat(files, list_file); } diff --git a/delete.c b/delete.c index 156beeee..1403b304 100644 --- a/delete.c +++ b/delete.c @@ -283,7 +283,7 @@ pgBackupDeleteFiles(pgBackup *backup) parray_qsort(files, pgFileComparePathDesc); for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(files, i); /* print progress */ elog(LOG, "delete file(%zd/%lu) \"%s\"", i + 1, diff --git a/dir.c b/dir.c index 71a32283..361b24e9 100644 --- a/dir.c +++ b/dir.c @@ -249,9 +249,6 @@ BlackListCompare(const void *str1, const void *str2) * List files, symbolic links and directories in the directory "root" and add * pgFile objects to "files". We add "root" to "files" if add_root is true. * - * If the sub-directory name is in "exclude" list, the sub-directory itself is - * listed but the contents of the sub-directory is ignored. - * * When omit_symlink is true, symbolic link is ignored and only file or * directory llnked to will be listed. */ @@ -259,40 +256,43 @@ void dir_list_file(parray *files, const char *root, bool exclude, bool omit_symlink, bool add_root) { - char path[MAXPGPATH]; - char buf[MAXPGPATH * 2]; - char black_item[MAXPGPATH * 2]; - parray *black_list = NULL; + parray *black_list = NULL; + char path[MAXPGPATH]; join_path_components(path, backup_path, PG_BLACK_LIST); - if (root && pgdata && strcmp(root, pgdata) == 0 && - fileExists(path)) + /* List files with black list */ + if (root && pgdata && strcmp(root, pgdata) == 0 && fileExists(path)) { - FILE *black_list_file = NULL; + FILE *black_list_file = NULL; + char buf[MAXPGPATH * 2]; + char black_item[MAXPGPATH * 2]; + black_list = parray_new(); black_list_file = fopen(path, "r"); + if (black_list_file == NULL) - elog(ERROR, "cannot open black_list: %s", - strerror(errno)); + elog(ERROR, "cannot open black_list: %s", strerror(errno)); + while (fgets(buf, lengthof(buf), black_list_file) != NULL) { join_path_components(black_item, pgdata, buf); + if (black_item[strlen(black_item) - 1] == '\n') black_item[strlen(black_item) - 1] = '\0'; + if (black_item[0] == '#' || black_item[0] == '\0') continue; + parray_append(black_list, black_item); } + fclose(black_list_file); parray_qsort(black_list, BlackListCompare); - dir_list_file_internal(files, root, exclude, omit_symlink, add_root, black_list); - parray_qsort(files, pgFileComparePath); - } - else - { - dir_list_file_internal(files, root, exclude, omit_symlink, add_root, NULL); - parray_qsort(files, pgFileComparePath); } + + dir_list_file_internal(files, root, exclude, omit_symlink, add_root, + black_list); + parray_qsort(files, pgFileComparePath); } void @@ -561,7 +561,8 @@ dir_print_file_list(FILE *out, const parray *files, const char *root, const char fprintf(out, " %s", timestamp); } - fprintf(out, " %d %d\n", file->generation, file->is_partial_copy); + fprintf(out, " " UINT64_FORMAT " %d\n", + file->generation, file->is_partial_copy); } } @@ -681,30 +682,34 @@ dir_read_file_list(const char *root, const char *file_txt) return files; } -/* copy contents of directory from_root into to_root */ +/* + * Copy contents of directory from_root into to_root. + */ void dir_copy_files(const char *from_root, const char *to_root) { - int i; - parray *files = parray_new(); + size_t i; + parray *files = parray_new(); /* don't copy root directory */ dir_list_file(files, from_root, false, true, false); for (i = 0; i < parray_num(files); i++) { - pgFile *file = (pgFile *) parray_get(files, i); + pgFile *file = (pgFile *) parray_get(files, i); if (S_ISDIR(file->mode)) { - char to_path[MAXPGPATH]; - join_path_components(to_path, to_root, file->path + strlen(from_root) + 1); + char to_path[MAXPGPATH]; + + join_path_components(to_path, to_root, + file->path + strlen(from_root) + 1); + if (verbose && !check) elog(LOG, "creating directory \"%s\"", file->path + strlen(from_root) + 1); if (!check) dir_create_dir(to_path, DIR_PERMISSION); - continue; } else if (S_ISREG(file->mode)) { diff --git a/pg_probackup.h b/pg_probackup.h index 9c5dce57..4ecac1bf 100644 --- a/pg_probackup.h +++ b/pg_probackup.h @@ -70,10 +70,10 @@ typedef struct pgFile char *path; /* path of the file */ char *ptrack_path; int segno; /* Segment number for ptrack */ - int generation; /* Generation of compressed file. + uint64 generation; /* Generation of compressed file. * -1 for non-compressed files */ - int is_partial_copy; /* for compressed files. - * 1 if backed up via copy_file_partly() */ + int is_partial_copy; /* for compressed files. + * 1 if backed up via copy_file_partly() */ volatile uint32 lock; datapagemap_t pagemap; } pgFile; diff --git a/tests/retention_test.py b/tests/retention_test.py index 5a4ef3a3..5a8e69fa 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -63,18 +63,29 @@ class RetentionTest(ProbackupTest, unittest.TestCase): self.init_pb(node) with open(path.join(self.backup_dir(node), "pg_probackup.conf"), "a") as conf: conf.write("REDUNDANCY=1\n") - conf.write("WINDOW=2\n") + conf.write("WINDOW=1\n") - # All backups will be keeped + # Make backups to be purged self.backup_pb(node) self.backup_pb(node, backup_type="page") + # Make backup to be keeped self.backup_pb(node) + + backups = path.join(self.backup_dir(node), "backups") + days_delta = 5 + for backup in listdir(backups): + with open(path.join(backups, backup, "backup.conf"), "a") as conf: + conf.write("RECOVERY_TIME='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta))) + days_delta -= 1 + + # Make backup to be keeped self.backup_pb(node, backup_type="page") self.assertEqual(len(self.show_pb(node)), 4) # Purge backups self.retention_purge_pb(node) - self.assertEqual(len(self.show_pb(node)), 4) + self.assertEqual(len(self.show_pb(node)), 2) node.stop() diff --git a/util.c b/util.c index 85266a5c..092830eb 100644 --- a/util.c +++ b/util.c @@ -72,20 +72,6 @@ digestControlFile(ControlFileData *ControlFile, char *src, size_t size) checkControlFile(ControlFile); } -void -sanityChecks(void) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata, "global/pg_control", &size, false); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - -} - XLogRecPtr get_last_ptrack_lsn(void) { @@ -114,8 +100,9 @@ get_current_timeline(bool safe) /* First fetch file... */ buffer = slurpFile(pgdata, "global/pg_control", &size, safe); - if (buffer == NULL) + if (safe && buffer == NULL) return 0; + digestControlFile(&ControlFile, buffer, size); pg_free(buffer);