db/pg_probackup
1
0
Fork 0
mirror of https://github.com/postgrespro/pg_probackup.git synced 2026-06-21 01:34:15 +02:00
Code Issues Releases Activity

[PBCKP-416] pioOpenDir + pioDirNext

Browse Source 
Looks like pioListDir were no so great idea. Step to return to iterator
style.
This commit is contained in:
Yura Sokolov
2022-12-18 21:32:44 +03:00
parent abdfdd2777
commit e79e5f0728
2 changed files with 323 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/file.c
+305 -3
View File
@@ -23,6 +23,8 @@ static char *async_errormsg = NULL;
#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64
static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0};
#define PIO_DIR_REMOTE_BATCH 100
fio_location MyLocation;
typedef struct
@@ -2736,6 +2738,61 @@ fio_communicate(int in, int out)
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
break;
}
case PIO_DIR_OPEN:
{
ft_assert(hdr.handle >= 0 && hdr.handle < FIO_FDMAX);
ft_assert(objs[hdr.handle] == NULL);
pioDirIter_i iter;
err_i err;
iter = $i(pioOpenDir, drive, buf, .err = &err);
if ($haserr(err))
fio_send_pio_err(out, err);
else
{
objs[hdr.handle] = $ref(iter.self);
hdr.size = 0;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
}
break;
}
case PIO_DIR_NEXT:
{
ft_assert(hdr.handle >= 0 && hdr.handle < FIO_FDMAX);
ft_assert(objs[hdr.handle] != NULL);
ft_strbuf_t stats = ft_strbuf_zero();
ft_strbuf_t names = ft_strbuf_zero();
pio_dirent_t dirent;
int n;
for (n = 0; n < PIO_DIR_REMOTE_BATCH; n++)
{
dirent = $(pioDirNext, objs[hdr.handle], .err = &err);
if ($haserr(err))
break;
if (dirent.stat.pst_kind == PIO_KIND_UNKNOWN)
break;
ft_strbuf_catbytes(&stats, FT_BYTES_FOR(dirent.stat));
ft_strbuf_cat_zt(&names, dirent.name);
}
if ($haserr(err))
fio_send_pio_err(out, err);
else
{
hdr.arg = n;
hdr.size = stats.len + names.len;
IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
if (n > 0)
{
IO_CHECK(fio_write_all(out, stats.ptr, stats.len), stats.len);
IO_CHECK(fio_write_all(out, names.ptr, names.len), names.len);
}
}
ft_strbuf_free(&stats);
ft_strbuf_free(&names);
break;
}
case PIO_CLOSE:
{
err_i err;
@@ -2827,6 +2884,7 @@ typedef struct pioLocalDir
{
ft_str_t path;
DIR* dir;
ft_strbuf_t name_buf;
} pioLocalDir;
#define kls__pioLocalDir iface__pioDirIter, iface(pioDirIter), mth(fobjDispose)
fobj_klass(pioLocalDir);
@@ -2857,6 +2915,21 @@ typedef struct pioRemoteWriteFile {
iface(pioWriteCloser, pioDBWriter)
fobj_klass(pioRemoteWriteFile);
#define FT_SLICE dirent
#define FT_SLICE_TYPE pio_dirent_t
#include <ft_array.inc.h>
typedef struct pioRemoteDir
{
ft_str_t path;
int handle;
int pos;
ft_bytes_t names_buf;
ft_arr_dirent_t entries;
} pioRemoteDir;
#define kls__pioRemoteDir iface__pioDirIter, iface(pioDirIter), mth(fobjDispose)
fobj_klass(pioRemoteDir);
typedef struct pioReadFilter {
pioRead_i wrapped;
pioFilter_i filter;
@@ -3241,6 +3314,26 @@ pioLocalDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_tab
skip_hidden, external_dir_num, $bind(pioDBDrive, self));
}
static pioDirIter_i
pioLocalDrive_pioOpenDir(VSelf, path_t path, err_i* err)
{
Self(pioLocalDrive);
DIR* dir;
fobj_reset_err(err);
dir = opendir(path);
if (dir == NULL)
{
*err = $syserr(errno, "Cannot open dir {path:q}", path(path));
return $null(pioDirIter);
}
return $bind(pioDirIter,
$alloc(pioLocalDir,
.path = ft_strdupc(path),
.dir = dir));
}
static void
pioLocalDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) {
FOBJ_FUNC_ARP();
@@ -3635,6 +3728,82 @@ pioLocalWriteFile_fobjDispose(VSelf)
ft_bytes_free(&self->buf);
}
static pio_dirent_t
pioLocalDir_pioDirNext(VSelf, err_i* err)
{
Self(pioLocalDir);
struct dirent* ent;
pio_dirent_t entry = {.stat={.pst_kind=PIO_KIND_UNKNOWN}};
char path[MAXPGPATH];
fobj_reset_err(err);
ft_assert(self->dir != NULL, "Abuse closed dir");
ft_strbuf_reset_for_reuse(&self->name_buf);
for (;;)
{
errno = 0;
ent = readdir(self->dir);
if (ent == NULL && errno != 0)
*err = $syserr(errno, "Could not read dir {path:q}",
path(self->path.ptr));
if (ent == NULL)
return entry;
/* Skip '.', '..' and all hidden files as well */
if (ent->d_name[0] == '.')
continue;
join_path_components(path, self->path.ptr, ent->d_name);
entry.stat = $i(pioStat, localDrive, path, true, .err = err);
if ($haserr(*err))
return entry;
/*
* Add only files, directories and links. Skip sockets and other
* unexpected file formats.
*/
if (entry.stat.pst_kind != PIO_KIND_DIRECTORY &&
entry.stat.pst_kind != PIO_KIND_REGULAR)
{
elog(WARNING, "Skip '%s': unexpected file kind %s", path,
pio_file_kind2str(entry.stat.pst_kind, path));
continue;
}
ft_strbuf_catc(&self->name_buf, ent->d_name);
entry.name = ft_strbuf_ref(&self->name_buf);
return entry;
}
}
static err_i
pioLocalDir_pioClose(VSelf)
{
Self(pioLocalDir);
int rc;
rc = closedir(self->dir);
self->dir = NULL;
if (rc)
return $syserr(errno, "Could not close dir {path:q}",
path(self->path.ptr));
return $noerr();
}
static void
pioLocalDir_fobjDispose(VSelf)
{
Self(pioLocalDir);
if (self->dir)
closedir(self->dir);
self->dir = NULL;
ft_str_free(&self->path);
ft_strbuf_free(&self->name_buf);
}
/* REMOTE DRIVE */
static pioReader_i
@@ -3727,9 +3896,8 @@ pioRemoteDrive_pioFilesAreSame(VSelf, path_t file1, path_t file2)
};
char _buf[512];
ft_strbuf_t buf = ft_strbuf_init_stack(_buf, sizeof(_buf));
ft_strbuf_catc(&buf, file1);
ft_strbuf_cat1(&buf, '\x00');
ft_strbuf_catc(&buf, file2);
ft_strbuf_catc_zt(&buf, file1);
ft_strbuf_catc_zt(&buf, file2);
hdr.size = buf.len + 1;
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
@@ -3923,6 +4091,35 @@ pioRemoteDrive_pioListDir(VSelf, parray *files, const char *root, bool handle_ta
pg_free(buf);
}
static pioDirIter_i
pioRemoteDrive_pioOpenDir(VSelf, path_t path, err_i* err)
{
Self(pioRemoteDrive);
fio_header hdr = {
.cop = PIO_DIR_OPEN,
.handle = find_free_handle(),
.size = strlen(path)+1,
};
fobj_reset_err(err);
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size);
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
if (hdr.cop == FIO_PIO_ERROR)
{
*err = fio_receive_pio_err(&hdr);
return $null(pioDirIter);
}
ft_assert(hdr.cop == PIO_DIR_OPEN);
set_handle(hdr.handle);
return $bind(pioDirIter,
$alloc(pioRemoteDir,
.path = ft_strdupc(path),
.handle = hdr.handle,
.pos = 0));
}
static void
pioRemoteDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) {
FOBJ_FUNC_ARP();
@@ -4565,6 +4762,109 @@ pioRemoteWriteFile_fobjDispose(VSelf)
ft_str_free(&self->path);
}
static pio_dirent_t
pioRemoteDir_pioDirNext(VSelf, err_i *err)
{
Self(pioRemoteDir);
fio_header hdr;
pio_dirent_t entry = {.stat={.pst_kind=PIO_KIND_UNKNOWN}};
ft_bytes_t tofree = ft_bytes(NULL, 0);
ft_bytes_t buf;
int n;
fobj_reset_err(err);
ft_assert(self->handle >= 0, "Abuse closed dir");
if (self->pos == self->entries.len)
{
ft_bytes_free(&self->names_buf);
ft_arr_dirent_reset_for_reuse(&self->entries);
self->pos = 0;
hdr = (fio_header){
.cop = PIO_DIR_NEXT,
.handle = self->handle,
};
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
if (hdr.cop == FIO_PIO_ERROR)
{
*err = fio_receive_pio_err(&hdr);
return entry;
}
ft_assert(hdr.cop == PIO_DIR_NEXT);
if (hdr.arg == 0)
{
/* End of iteration */
return entry;
}
buf = ft_bytes_alloc(hdr.size);
tofree = buf;
IO_CHECK(fio_read_all(fio_stdin, buf.ptr, buf.len), buf.len);
for (n = 0; n < hdr.arg; n++)
{
ft_bytes_shift_must(&buf, FT_BYTES_FOR(entry.stat));
ft_arr_dirent_push(&self->entries, entry);
}
self->names_buf = ft_bytes_dup(buf);
buf = self->names_buf;
for (n = 0; n < self->entries.len; n++)
self->entries.ptr[n].name = ft_bytes_shift_zt(&buf);
ft_bytes_free(&tofree);
}
entry = self->entries.ptr[self->pos];
self->pos++;
return entry;
}
static err_i
pioRemoteDir_pioClose(VSelf)
{
Self(pioRemoteDir);
err_i err = $noerr();
fio_header hdr = {.cop = PIO_CLOSE, .handle = self->handle };
ft_assert(self->handle >= 0);
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
unset_handle(self->handle);
self->handle = -1;
if (hdr.cop == FIO_PIO_ERROR)
err = fobj_err_combine(err, fio_receive_pio_err(&hdr));
return err;
}
static void
pioRemoteDir_fobjDispose(VSelf)
{
Self(pioRemoteDir);
if (self->handle >= 0)
{
fio_header hdr = {
.cop = PIO_DISPOSE,
.handle = self->handle,
};
IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
unset_handle(self->handle);
}
ft_str_free(&self->path);
ft_bytes_free(&self->names_buf);
ft_arr_dirent_reset_for_reuse(&self->entries);
}
pioRead_i
pioWrapReadFilter(pioRead_i fl, pioFilter_i flt, size_t buf_size)
{
@@ -5922,6 +6222,8 @@ fobj_klass_handle(pioLocalFile, inherits(pioFile), mth(fobjDispose, fobjRepr));
fobj_klass_handle(pioRemoteFile, inherits(pioFile), mth(fobjDispose, fobjRepr));
fobj_klass_handle(pioLocalWriteFile);
fobj_klass_handle(pioRemoteWriteFile);
fobj_klass_handle(pioLocalDir);
fobj_klass_handle(pioRemoteDir);
fobj_klass_handle(pioWriteFilter, mth(fobjDispose, fobjRepr));
fobj_klass_handle(pioReadFilter, mth(fobjDispose, fobjRepr));
fobj_klass_handle(pioDevNull);
src/utils/file.h
+18 -3
View File
@@ -70,6 +70,8 @@ typedef enum
PIO_SEEK,
PIO_TRUNCATE,
PIO_GET_ASYNC_ERROR,
PIO_DIR_OPEN,
PIO_DIR_NEXT,
PIO_CLOSE,
PIO_DISPOSE,
} fio_operations;
@@ -108,12 +110,17 @@ typedef enum pio_file_kind {
} pio_file_kind_e;
typedef struct pio_stat {
pio_file_kind_e pst_kind;
uint32_t pst_mode;
int64_t pst_size;
int64_t pst_mtime;
uint32_t pst_mode;
pio_file_kind_e pst_kind;
} pio_stat_t;
typedef struct pio_dirent {
pio_stat_t stat;
ft_str_t name;
} pio_dirent_t;
extern fio_location MyLocation;
extern void setMyLocation(ProbackupSubcmd const subcmd);
@@ -229,6 +236,12 @@ fobj_iface(pioWriteCloser);
fobj_iface(pioDBWriter);
fobj_iface(pioReadCloser);
// DIR
#define mth__pioDirNext pio_dirent_t, (err_i*, err)
fobj_method(pioDirNext);
#define iface__pioDirIter mth(pioDirNext, pioClose)
fobj_iface(pioDirIter);
// Pages iterator
typedef struct
{
@@ -270,6 +283,7 @@ fobj_iface(pioPagesIterator);
#define mth__pioFilesAreSame bool, (path_t, file1), (path_t, file2)
#define mth__pioIsRemote bool
#define mth__pioMakeDir err_i, (path_t, path), (mode_t, mode), (bool, strict)
#define mth__pioOpenDir pioDirIter_i, (path_t, path), (err_i*, err)
#define mth__pioListDir void, (parray *, files), (const char *, root), \
(bool, handle_tablespaces), (bool, symlink_and_hidden), \
(bool, backup_logs), (bool, skip_hidden), (int, external_dir_num)
@@ -298,6 +312,7 @@ fobj_method(pioIsRemote);
fobj_method(pioGetCRC32);
fobj_method(pioMakeDir);
fobj_method(pioFilesAreSame);
fobj_method(pioOpenDir);
fobj_method(pioListDir);
fobj_method(pioRemoveDir);
fobj_method(pioReadFile);
@@ -307,7 +322,7 @@ fobj_method(pioIteratePages);
#define iface__pioDrive mth(pioOpenRead, pioOpenReadStream), \
mth(pioStat, pioRemove), \
mth(pioExists, pioGetCRC32, pioIsRemote), \
mth(pioMakeDir, pioListDir, pioRemoveDir), \
mth(pioMakeDir, pioOpenDir, pioListDir, pioRemoveDir), \
mth(pioFilesAreSame, pioReadFile, pioWriteFile), \
mth(pioOpenRewrite)
fobj_iface(pioDrive);