Merge branch 'REL_2_6_slurp' into 'REL_2_6'

[PBCKP-349] pioReadFile replaces slurpFile

See merge request eepbk/pg_probackup_ee!1

src/pg_probackup.h

@@ -1123,11 +1123,10 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo	segno, T
 
 /* in util.c */
 extern TimeLineID get_current_timeline(PGconn *conn);
-extern TimeLineID get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe);
+extern TimeLineID get_current_timeline_from_control(fio_location location, const char *pgdata_path);
 extern XLogRecPtr get_checkpoint_location(PGconn *conn);
 extern uint64 get_system_identifier(fio_location location, const char *pgdata_path, bool safe);
 extern uint64 get_remote_system_identifier(PGconn *conn);
-extern uint32 get_data_checksum_version(bool safe);
 extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);
 extern uint32 get_xlog_seg_size(const char *pgdata_path);
 extern void get_redo(fio_location location, const char *pgdata_path, RedoParams *redo);

src/util.c

@@ -96,24 +96,6 @@ checkControlFile(ControlFileData *ControlFile)
 			 "the PostgreSQL installation would be incompatible with this data directory.");
 }
 
-/*
- * Verify control file contents in the buffer src, and copy it to *ControlFile.
- */
-static void
-digestControlFile(ControlFileData *ControlFile, char *src, size_t size)
-{
-	int			ControlFileSize = PG_CONTROL_FILE_SIZE;
-
-	if (size != ControlFileSize)
-		elog(ERROR, "unexpected control file size %d, expected %d",
-			 (int) size, ControlFileSize);
-
-	memcpy(ControlFile, src, sizeof(ControlFileData));
-
-	/* Additional checks on control file */
-	checkControlFile(ControlFile);
-}
-
 /*
  * Write ControlFile to pg_control
  */
@@ -164,7 +146,7 @@ get_current_timeline(PGconn *conn)
 	if (PQresultStatus(res) == PGRES_TUPLES_OK)
 		val = PQgetvalue(res, 0, 0);
 	else
-		return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false);
+		return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata);
 
 	if (!parse_uint32(val, &tli, 0))
 	{
@@ -172,28 +154,62 @@ get_current_timeline(PGconn *conn)
 		elog(WARNING, "Invalid value of timeline_id %s", val);
 
 		/* TODO 3.0 remove it and just error out */
-		return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata, false);
+		return get_current_timeline_from_control(FIO_DB_HOST, instance_config.pgdata);
 	}
 
 	return tli;
 }
 
+static err_i
+get_control_file(fio_location location, path_t pgdata_path, path_t file,
+				 ControlFileData *control, bool safe)
+{
+	pioDrive_i	drive;
+	char		fullpath[MAXPGPATH];
+	ft_bytes_t	bytes;
+	err_i		err;
+
+	fobj_reset_err(&err);
+
+	join_path_components(fullpath, pgdata_path, file);
+
+	drive = pioDriveForLocation(location);
+	bytes = $i(pioReadFile, drive, .path = fullpath, .err = &err);
+	if ($haserr(err) && safe)
+	{
+		ft_logerr(FT_WARNING, $errmsg(err), "Could not get control file");
+		memset(control, 0, sizeof(ControlFileData));
+		return $noerr();
+	}
+	if ($haserr(err))
+		return $err(RT, "Could not get control file: {cause}",
+					cause(err.self));
+
+	if (bytes.len != PG_CONTROL_FILE_SIZE)
+		return $err(RT, "unexpected control file size: {size}, expected {wantedSz}",
+					size(bytes.len), wantedSz(PG_CONTROL_FILE_SIZE));
+
+	memcpy(control, bytes.ptr, sizeof(ControlFileData));
+	ft_bytes_free(&bytes);
+
+	/* Additional checks on control file */
+	checkControlFile(control);
+
+	return $noerr();
+}
+
 /* Get timeline from pg_control file */
 TimeLineID
-get_current_timeline_from_control(fio_location location, const char *pgdata_path, bool safe)
+get_current_timeline_from_control(fio_location location, const char *pgdata_path)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char       *buffer;
-	size_t      size;
+	err_i			err;
 
-	/* First fetch file... */
-	buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE,
-					   &size, safe);
-	if (safe && buffer == NULL)
-		return 0;
-
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
+	err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE,
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Getting current timeline");
 
 	return ControlFile.checkPointCopy.ThisTimeLineID;
 }
@@ -223,16 +239,14 @@ get_checkpoint_location(PGconn *conn)
 uint64
 get_system_identifier(fio_location location, const char *pgdata_path, bool safe)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
+	err_i			err;
 
-	/* First fetch file... */
-	buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, safe);
-	if (safe && buffer == NULL)
-		return 0;
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
+	err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE,
+						   &ControlFile, safe);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Getting system identifier");
 
 	return ControlFile.system_identifier;
 }
@@ -262,14 +276,14 @@ uint32
 get_xlog_seg_size(const char *pgdata_path)
 {
 #if PG_VERSION_NUM >= 110000
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
+	err_i			err;
 
-	/* First fetch file... */
-	buffer = slurpFile(FIO_DB_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false);
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
+	err = get_control_file(FIO_DB_HOST, pgdata_path, XLOG_CONTROL_FILE,
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Trying to fetch segment size");
 
 	return ControlFile.xlog_seg_size;
 #else
@@ -277,36 +291,17 @@ get_xlog_seg_size(const char *pgdata_path)
 #endif
 }
 
-uint32
-get_data_checksum_version(bool safe)
-{
-	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
-
-	/* First fetch file... */
-	buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE,
-					   &size, safe);
-	if (buffer == NULL)
-		return 0;
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
-
-	return ControlFile.data_checksum_version;
-}
-
 pg_crc32c
 get_pgcontrol_checksum(const char *pgdata_path)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
+	err_i			err;
 
-	/* First fetch file... */
-	buffer = slurpFile(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE, &size, false);
-	elog(WARNING, "checking %s", pgdata_path);
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
+	err = get_control_file(FIO_BACKUP_HOST, pgdata_path, XLOG_CONTROL_FILE,
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Getting pgcontrol checksum");
 
 	return ControlFile.crc;
 }
@@ -314,15 +309,14 @@ get_pgcontrol_checksum(const char *pgdata_path)
 void
 get_redo(fio_location location, const char *pgdata_path, RedoParams *redo)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
+	err_i			err;
 
-	/* First fetch file... */
-	buffer = slurpFile(location, pgdata_path, XLOG_CONTROL_FILE, &size, false);
-
-	digestControlFile(&ControlFile, buffer, size);
-	pg_free(buffer);
+	err = get_control_file(location, pgdata_path, XLOG_CONTROL_FILE,
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Fetching redo lsn");
 
 	redo->lsn = ControlFile.checkPointCopy.redo;
 	redo->tli = ControlFile.checkPointCopy.ThisTimeLineID;
@@ -352,14 +346,15 @@ void
 set_min_recovery_point(pgFile *file, const char *backup_path,
 					   XLogRecPtr stop_backup_lsn)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char       *buffer;
-	size_t      size;
-	char		fullpath[MAXPGPATH];
+	char			fullpath[MAXPGPATH];
+	err_i			err;
 
-	/* First fetch file content */
-	buffer = slurpFile(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE, &size, false);
-	digestControlFile(&ControlFile, buffer, size);
+	err = get_control_file(FIO_DB_HOST, instance_config.pgdata, XLOG_CONTROL_FILE,
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Set min recovery point");
 
 	elog(LOG, "Current minRecPoint %X/%X",
 		(uint32) (ControlFile.minRecoveryPoint  >> 32),
@@ -383,8 +378,6 @@ set_min_recovery_point(pgFile *file, const char *backup_path,
 
 	/* Update pg_control checksum in backup_list */
 	file->crc = ControlFile.crc;
-
-	pg_free(buffer);
 }
 
 /*
@@ -394,22 +387,21 @@ void
 copy_pgcontrol_file(fio_location from_location, const char *from_fullpath,
 					fio_location to_location, const char *to_fullpath, pgFile *file)
 {
+	FOBJ_FUNC_ARP();
 	ControlFileData ControlFile;
-	char	   *buffer;
-	size_t		size;
+	err_i			err;
 
-	buffer = slurpFile(from_location, from_fullpath, "", &size, false);
-
-	digestControlFile(&ControlFile, buffer, size);
+	err = get_control_file(from_location, from_fullpath, "",
+						   &ControlFile, false);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Fetching control file");
 
 	file->crc = ControlFile.crc;
-	file->read_size = (int64_t)size;
-	file->write_size = (int64_t)size;
-	file->uncompressed_size = (int64_t)size;
+	file->read_size = PG_CONTROL_FILE_SIZE;
+	file->write_size = PG_CONTROL_FILE_SIZE;
+	file->uncompressed_size = PG_CONTROL_FILE_SIZE;
 
 	writeControlFile(to_location, to_fullpath, &ControlFile);
-
-	pg_free(buffer);
 }
 
 /*

src/utils/file.c

@@ -3600,6 +3600,7 @@ fio_communicate(int in, int out)
 	fio_header hdr;
 	pioDrive_i drive;
 	pio_stat_t st;
+	ft_bytes_t bytes;
 	int rc;
 	int tmp_fd;
 	pg_crc32 crc;
@@ -3720,6 +3721,27 @@ fio_communicate(int in, int out)
 			hdr.size = 0;
 			IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
 			break;
+		  case FIO_READ_FILE_AT_ONCE:
+			bytes = $i(pioReadFile, drive, .path = buf,
+					  .binary = hdr.arg != 0, .err = &err);
+			if ($haserr(err))
+			{
+				const char *msg = $errmsg(err);
+				hdr.arg = getErrno(err);
+				hdr.size = strlen(msg) + 1;
+				IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
+				IO_CHECK(fio_write_all(out, msg, hdr.size), hdr.size);
+			}
+			else
+			{
+				hdr.arg = 0;
+				hdr.size = bytes.len;
+				IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr));
+				if (bytes.len > 0)
+					IO_CHECK(fio_write_all(out, bytes.ptr, bytes.len), bytes.len);
+			}
+			ft_bytes_free(&bytes);
+			break;
 		  case FIO_ACCESS: /* Check presence of file with specified name */
 			hdr.size = 0;
 			hdr.arg = access(buf, hdr.arg) < 0 ? errno  : 0;
@@ -4151,6 +4173,89 @@ pioLocalDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) {
     parray_free(files);
 }
 
+static ft_bytes_t
+pioLocalDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err)
+{
+	FOBJ_FUNC_ARP();
+	Self(pioLocalDrive);
+	pioFile_i	fl;
+	pio_stat_t	st;
+	ft_bytes_t	res = ft_bytes(NULL, 0);
+	size_t		amount;
+
+	fobj_reset_err(err);
+
+	st = $(pioStat, self, .path = path, .follow_symlink = true, .err = err);
+	if ($haserr(*err))
+	{
+		$iresult(*err);
+		return res;
+	}
+	if (st.pst_kind != PIO_KIND_REGULAR)
+	{
+		*err = $err(RT, "File {path:q} is not regular: {kind}", path(path),
+					kind(pio_file_kind2str(st.pst_kind, path)));
+		$iresult(*err);
+		return res;
+	}
+
+	/* forbid too large file because of remote protocol */
+	if (st.pst_size >= INT32_MAX)
+	{
+		*err = $err(RT, "File {path:q} is too large: {size}", path(path),
+					size(st.pst_size), errNo(ENOMEM));
+		$iresult(*err);
+		return res;
+	}
+	if (binary)
+		res = ft_bytes_alloc(st.pst_size);
+	else
+	{
+		res = ft_bytes_alloc(st.pst_size + 1);
+		res.len -= 1;
+	}
+
+	/*
+	 * rely on "local file is read whole at once always".
+	 * Is it true?
+	 */
+	fl = $(pioOpen, self, .path = path, .flags = O_RDONLY | (binary ? PG_BINARY : 0),
+		   .err = err);
+	if ($haserr(*err))
+	{
+		$iresult(*err);
+		return res;
+	}
+
+	amount = pioReadFull($reduce(pioRead, fl), res, err);
+	if ($haserr(*err))
+	{
+		ft_bytes_free(&res);
+		$iresult(*err);
+		return res;
+	}
+
+	if (amount != st.pst_size)
+	{
+		ft_bytes_free(&res);
+		*err = $err(RT, "File {path:q} is truncated while reading",
+					path(path), errNo(EBUSY));
+		$iresult(*err);
+		return res;
+	}
+
+	if (binary)
+		res.len = amount;
+	else
+	{
+		res.len = amount + 1;
+		res.ptr[amount] = 0;
+	}
+
+	$i(pioClose, fl);
+	return res;
+}
+
 /* LOCAL FILE */
 static void
 pioLocalFile_fobjDispose(VSelf)
@@ -4564,6 +4669,43 @@ pioRemoteDrive_pioRemoveDir(VSelf, const char *root, bool root_as_well) {
         elog(ERROR, "couldn't remove remote dir");
 }
 
+static ft_bytes_t
+pioRemoteDrive_pioReadFile(VSelf, path_t path, bool binary, err_i* err)
+{
+	FOBJ_FUNC_ARP();
+	Self(pioLocalDrive);
+	ft_bytes_t res;
+
+	fobj_reset_err(err);
+
+	fio_header hdr = {
+			.cop = FIO_READ_FILE_AT_ONCE,
+			.handle = -1,
+			.size = strlen(path)+1,
+			.arg = binary,
+	};
+
+	IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr));
+	IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size);
+
+	/* get the response */
+	IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr));
+	Assert(hdr.cop == FIO_READ_FILE_AT_ONCE);
+
+	res = ft_bytes_alloc(hdr.size);
+	IO_CHECK(fio_read_all(fio_stdin, res.ptr, hdr.size), hdr.size);
+
+	if (hdr.arg != 0)
+	{
+		*err = $syserr((int)hdr.arg, "Could not read remote file {path:q}: {causeStr}",
+					   path(path), causeStr(res.ptr));
+		$iresult(*err);
+		ft_bytes_free(&res);
+	}
+
+	return res;
+}
+
 /* REMOTE FILE */
 
 static err_i
@@ -5528,6 +5670,25 @@ pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src,
     return $noerr();
 }
 
+size_t
+pioReadFull(pioRead_i src, ft_bytes_t bytes, err_i* err)
+{
+	ft_bytes_t	b;
+	size_t		r;
+	fobj_reset_err(err);
+
+	b = bytes;
+	while (b.len)
+	{
+		r = $i(pioRead, src, b, err);
+		Assert(r <= b.len);
+		ft_bytes_consume(&b, r);
+		if ($haserr(*err))
+			break;
+	}
+	return bytes.len - b.len;
+}
+
 fobj_klass_handle(pioFile);
 fobj_klass_handle(pioLocalDrive);
 fobj_klass_handle(pioRemoteDrive);

src/utils/file.h

@@ -69,6 +69,7 @@ typedef enum
 	FIO_SEND_FILE_CONTENT,
 	FIO_PAGE_ZERO,
 	FIO_FILES_ARE_SAME,
+	FIO_READ_FILE_AT_ONCE,
 } fio_operations;
 
 typedef struct
@@ -227,6 +228,8 @@ typedef const char* path_t;
 fobj_error_cstr_key(remotemsg);
 fobj_error_int_key(writtenSz);
 fobj_error_int_key(wantedSz);
+fobj_error_int_key(size);
+fobj_error_cstr_key(kind);
 
 #ifdef HAVE_LIBZ
 fobj_error_kind(GZ);
@@ -275,6 +278,9 @@ fobj_iface(pioReadCloser);
                                 (bool, handle_tablespaces), (bool, symlink_and_hidden), \
                                 (bool, backup_logs), (bool, skip_hidden),  (int, external_dir_num)
 #define mth__pioRemoveDir   void, (const char *, root), (bool, root_as_well)
+#define mth__pioReadFile	ft_bytes_t, (path_t, path), (bool, binary), \
+							(err_i *, err)
+#define mth__pioReadFile__optional() (binary, true)
 
 fobj_method(pioOpen);
 fobj_method(pioStat);
@@ -287,11 +293,12 @@ fobj_method(pioMakeDir);
 fobj_method(pioFilesAreSame);
 fobj_method(pioListDir);
 fobj_method(pioRemoveDir);
+fobj_method(pioReadFile);
 
 #define iface__pioDrive 	mth(pioOpen, pioStat, pioRemove, pioRename), \
 					        mth(pioExists, pioGetCRC32, pioIsRemote),                \
 							mth(pioMakeDir, pioListDir, pioRemoveDir),  \
-							mth(pioFilesAreSame)
+							mth(pioFilesAreSame), mth(pioReadFile)
 fobj_iface(pioDrive);
 
 extern pioDrive_i pioDriveForLocation(fio_location location);
@@ -340,4 +347,6 @@ extern err_i    pioCopyWithFilters(pioWriteFlush_i dest, pioRead_i src,
         pioFilter_i _fltrs_[] = {__VA_ARGS__}; \
         pioCopyWithFilters((dest), (src), _fltrs_, ft_arrsz(_fltrs_), NULL); \
 })
+
+extern size_t	pioReadFull(pioRead_i src, ft_bytes_t bytes, err_i* err);
 #endif