[PBCKP-351] add new way to parse config options

It is prerequisite for killing fio_open_stream in get_backup_list.

src/catalog.c

@@ -1019,6 +1019,7 @@ get_backup_filelist(pgBackup *backup, bool strict)
 	char     buf[BLCKSZ];
 	char     stdio_buf[STDIO_BUFSIZE];
 	pg_crc32 content_crc = 0;
+	pb_control_line	pb_line;
 
 	join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST);
 
@@ -1034,12 +1035,14 @@ get_backup_filelist(pgBackup *backup, bool strict)
 
 	INIT_CRC32C(content_crc);
 
+	init_pb_control_line(&pb_line);
+
 	while (fgets(buf, lengthof(buf), fp))
 	{
-		char		path[MAXPGPATH];
-		char		linked[MAXPGPATH];
-		char		compress_alg_string[MAXPGPATH];
-		char 		kind[16];
+		ft_str_t 	path;
+		ft_str_t 	linked;
+		ft_str_t 	compress_alg;
+		ft_str_t 	kind;
 		int64		write_size,
 					uncompressed_size,
 					mode,		/* bit length of mode_t depends on platforms */
@@ -1058,59 +1061,66 @@ get_backup_filelist(pgBackup *backup, bool strict)
 
 		COMP_CRC32C(content_crc, buf, strlen(buf));
 
-		get_control_value_str(buf, "path", path, sizeof(path),true);
-		get_control_value_int64(buf, "size", &write_size, true);
-		get_control_value_int64(buf, "mode", &mode, true);
-		get_control_value_int64(buf, "is_datafile", &is_datafile, true);
-		get_control_value_int64(buf, "is_cfs", &is_cfs, false);
-		get_control_value_int64(buf, "crc", &crc, true);
-		get_control_value_str(buf, "compress_alg", compress_alg_string, sizeof(compress_alg_string), false);
-		get_control_value_int64(buf, "external_dir_num", &external_dir_num, false);
-		get_control_value_int64(buf, "dbOid", &dbOid, false);
+		parse_pb_control_line(&pb_line, ft_str2bytes(ft_cstr(buf)));
 
-		file = pgFileInit(path);
+		path         = pb_control_line_get_str(&pb_line,   "path");
+		write_size   = pb_control_line_get_int64(&pb_line, "size");
+		mode         = pb_control_line_get_int64(&pb_line, "mode");
+		is_datafile  = pb_control_line_get_int64(&pb_line, "is_datafile");
+		crc          = pb_control_line_get_int64(&pb_line, "crc");
+
+		pb_control_line_try_int64(&pb_line, "is_cfs",       &is_cfs);
+		pb_control_line_try_int64(&pb_line, "dbOid",        &dbOid);
+		pb_control_line_try_str(&pb_line,   "compress_alg", &compress_alg);
+		pb_control_line_try_int64(&pb_line, "external_dir_num", &external_dir_num);
+
+		if (path.len > MAXPGPATH)
+			elog(ERROR, "File path in "DATABASE_FILE_LIST" is too long: '%s'",
+				 buf);
+
+		file = pgFileInit(path.ptr);
 		file->write_size = (int64) write_size;
 		file->mode = (mode_t) mode;
 		file->is_datafile = is_datafile ? true : false;
 		file->is_cfs = is_cfs ? true : false;
 		file->crc = (pg_crc32) crc;
-		file->compress_alg = parse_compress_alg(compress_alg_string);
-		file->external_dir_num = external_dir_num;
+		file->compress_alg = parse_compress_alg(compress_alg.ptr);
+		file->external_dir_num = (int)external_dir_num;
 		file->dbOid = dbOid ? dbOid : 0;
 
 		/*
 		 * Optional fields
 		 */
-		if (get_control_value_str(buf, "kind", kind, sizeof(kind), false))
-			file->kind = pio_str2file_kind(kind, path);
+		if (pb_control_line_try_str(&pb_line, "kind", &kind))
+			file->kind = pio_str2file_kind(kind.ptr, path.ptr);
 		else /* fallback to mode for old backups */
-			file->kind = pio_statmode2file_kind(file->mode, path);
+			file->kind = pio_statmode2file_kind(file->mode, path.ptr);
 
-		if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0])
+		if (pb_control_line_try_str(&pb_line, "linked", &linked) && linked.len > 0)
 		{
-			file->linked = pgut_strdup(linked);
+			file->linked = ft_strdup(linked).ptr;
 			canonicalize_path(file->linked);
 		}
 
-		if (get_control_value_int64(buf, "segno", &segno, false))
+		if (pb_control_line_try_int64(&pb_line, "segno", &segno))
 			file->segno = (int) segno;
 
-		if (get_control_value_int64(buf, "n_blocks", &n_blocks, false))
+		if (pb_control_line_try_int64(&pb_line, "n_blocks", &n_blocks))
 			file->n_blocks = (int) n_blocks;
 
-		if (get_control_value_int64(buf, "n_headers", &n_headers, false))
+		if (pb_control_line_try_int64(&pb_line, "n_headers", &n_headers))
 			file->n_headers = (int) n_headers;
 
-		if (get_control_value_int64(buf, "hdr_crc", &hdr_crc, false))
+		if (pb_control_line_try_int64(&pb_line, "hdr_crc", &hdr_crc))
 			file->hdr_crc = (pg_crc32) hdr_crc;
 
-		if (get_control_value_int64(buf, "hdr_off", &hdr_off, false))
+		if (pb_control_line_try_int64(&pb_line, "hdr_off", &hdr_off))
 			file->hdr_off = hdr_off;
 
-		if (get_control_value_int64(buf, "hdr_size", &hdr_size, false))
+		if (pb_control_line_try_int64(&pb_line, "hdr_size", &hdr_size))
 			file->hdr_size = (int) hdr_size;
 
-		if (get_control_value_int64(buf, "full_size", &uncompressed_size, false))
+		if (pb_control_line_try_int64(&pb_line, "full_size", &uncompressed_size))
 			file->uncompressed_size = uncompressed_size;
 		else
 			file->uncompressed_size = write_size;
@@ -1138,6 +1148,8 @@ get_backup_filelist(pgBackup *backup, bool strict)
 		parray_append(files, file);
 	}
 
+	deinit_pb_control_line(&pb_line);
+
 	FIN_CRC32C(content_crc);
 
 	if (ferror(fp))

src/dir.c

@@ -124,7 +124,7 @@ static void opt_path_map(ConfigOption *opt, const char *arg,
 						 TablespaceList *list, const char *type);
 static void cleanup_tablespace(const char *path);
 
-static void control_string_bad_format(const char* str);
+static void control_string_bad_format(ft_bytes_t str);
 
 static bool exclude_files_cb(void *value, void *exclude_args);
 
@@ -1135,160 +1135,183 @@ get_external_remap(char *current_dir)
 	return current_dir;
 }
 
-/* Parsing states for get_control_value_str() */
-#define CONTROL_WAIT_NAME			1
-#define CONTROL_INNAME				2
-#define CONTROL_WAIT_COLON			3
-#define CONTROL_WAIT_VALUE			4
-#define CONTROL_INVALUE				5
-#define CONTROL_WAIT_NEXT_NAME		6
-
 /*
- * Get value from json-like line "str" of backup_content.control file.
+ * Parse values from json-like line "str" of backup_content.control file.
  *
  * The line has the following format:
  *   {"name1":"value1", "name2":"value2"}
- *
- * The value will be returned in "value_int64" as int64.
- *
- * Returns true if the value was found in the line and parsed.
  */
-bool
-get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory)
+
+typedef struct pb_control_line_kv {
+	uint32_t key_hash;
+	uint32_t key_start;
+	uint32_t key_len;
+	uint32_t val_len;
+} pb_control_line_kv;
+
+#define FT_SLICE clkv
+#define FT_SLICE_TYPE pb_control_line_kv
+#include "ft_array.inc.h"
+
+void
+init_pb_control_line(pb_control_line* pb_line)
 {
+	pb_line->kvs = ft_malloc(sizeof(ft_arr_clkv_t));
+	*pb_line->kvs = (ft_arr_clkv_t)ft_arr_init();
+	pb_line->strbuf = ft_strbuf_zero();
+}
 
-	char buf_int64[32];
+void
+parse_pb_control_line(pb_control_line* pb_line, ft_bytes_t line)
+{
+	pb_control_line_kv kv = {0};
+	ft_strbuf_t		*strbuf = &pb_line->strbuf;
+	ft_bytes_t		parse;
 
-	assert(value_int64);
+	pb_line->line = line;
+	ft_arr_clkv_reset_for_reuse(pb_line->kvs);
+	ft_strbuf_reset_for_reuse(&pb_line->strbuf);
 
-    /* Set default value */
-    *value_int64 = 0;
+	parse = line;
+	ft_bytes_consume(&parse, ft_bytes_spnc(parse, "{ \t"));
+	while (parse.len)
+	{
+		ft_bytes_t name;
+		ft_bytes_t value;
 
-	if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory))
+		ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES));
+		/* name in quotes */
+		if (!ft_bytes_starts_withc(parse, "\""))
+			control_string_bad_format(line);
+		ft_bytes_consume(&parse, 1); /* skip quote */
+
+		name = ft_bytes_split(&parse, ft_bytes_notspnc(parse, "\""));
+		if (!ft_bytes_starts_withc(parse, "\""))
+			control_string_bad_format(line);
+		kv.key_start = strbuf->len;
+		kv.key_len = name.len;
+		ft_strbuf_catbytes(strbuf, name);
+		ft_strbuf_cat1(strbuf, '\0');
+		kv.key_hash = ft_small_cstr_hash(strbuf->ptr + kv.key_start);
+
+		ft_bytes_consume(&parse, 1); /* skip quote */
+		ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES));
+		if (!ft_bytes_starts_withc(parse, ":"))
+			control_string_bad_format(line);
+		ft_bytes_consume(&parse, 1); /* skip colon */
+		ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES));
+
+		/* value in quotes */
+		if (!ft_bytes_starts_withc(parse, "\""))
+			control_string_bad_format(line);
+		ft_bytes_consume(&parse, 1); /* skip quote */
+
+		value = ft_bytes_split(&parse, ft_bytes_notspnc(parse, "\""));
+		if (!ft_bytes_starts_withc(parse, "\""))
+			control_string_bad_format(line);
+		kv.val_len = value.len;
+		ft_strbuf_catbytes(strbuf, value);
+		ft_strbuf_cat1(strbuf, '\0');
+		ft_arr_clkv_push(pb_line->kvs, kv);
+
+		ft_bytes_consume(&parse, 1); /* skip quote */
+		ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES));
+		if (ft_bytes_starts_withc(parse, ","))
+		{
+			ft_bytes_consume(&parse, 1);
+			continue;
+		}
+		break;
+	}
+
+	if (!ft_bytes_starts_withc(parse, "}"))
+		control_string_bad_format(line);
+	ft_bytes_consume(&parse, 1);
+	ft_bytes_consume(&parse, ft_bytes_spnc(parse, SPACES));
+	if (parse.len != 0)
+		control_string_bad_format(line);
+}
+
+void
+deinit_pb_control_line(pb_control_line *pb_line)
+{
+	ft_arr_clkv_free(pb_line->kvs);
+	ft_free(pb_line->kvs);
+	pb_line->kvs = NULL;
+	ft_strbuf_free(&pb_line->strbuf);
+}
+
+bool
+pb_control_line_try_str(pb_control_line *pb_line, const char *name, ft_str_t *value)
+{
+	pb_control_line_kv kv;
+	ft_str_t key;
+	uint32_t i;
+	uint32_t key_hash = ft_small_cstr_hash(name);
+
+	for (i = 0; i < pb_line->kvs->len; i++)
+	{
+		kv = ft_arr_clkv_at(pb_line->kvs, i);
+		if (kv.key_hash != key_hash)
+			continue;
+		key = ft_str(pb_line->strbuf.ptr + kv.key_start, kv.key_len);
+		if (!ft_streqc(key, name))
+			continue;
+		*value = ft_str(pb_line->strbuf.ptr + kv.key_start + kv.key_len + 1,
+					    kv.val_len);
+		return true;
+	}
+	*value = ft_str("", 0);
+	return false;
+}
+
+bool
+pb_control_line_try_int64(pb_control_line *pb_line, const char *name, int64 *value)
+{
+	ft_str_t val;
+
+	*value = 0;
+	if (!pb_control_line_try_str(pb_line, name, &val))
 		return false;
 
-	if (!parse_int64(buf_int64, value_int64, 0))
+	if (!parse_int64(val.ptr, value, 0))
 	{
 		/* We assume that too big value is -1 */
 		if (errno == ERANGE)
-			*value_int64 = BYTES_INVALID;
+			*value = BYTES_INVALID;
 		else
-			control_string_bad_format(str);
-        return false;
+			control_string_bad_format(pb_line->line);
+		return false;
 	}
 
 	return true;
 }
 
-/*
- * Get value from json-like line "str" of backup_content.control file.
- *
- * The line has the following format:
- *   {"name1":"value1", "name2":"value2"}
- *
- * The value will be returned to "value_str" as string.
- *
- * Returns true if the value was found in the line.
- */
-
-bool
-get_control_value_str(const char *str, const char *name,
-                      char *value_str, size_t value_str_size, bool is_mandatory)
+ft_str_t
+pb_control_line_get_str(pb_control_line *pb_line, const char *name)
 {
-	int			state = CONTROL_WAIT_NAME;
-	char	   *name_ptr = (char *) name;
-	char	   *buf = (char *) str;
-	char 	   *const value_str_start = value_str;
+	ft_str_t res;
+	if (!pb_control_line_try_str(pb_line, name, &res))
+		elog(ERROR, "field \"%s\" is not found in the line %.*s of the file %s",
+			 name, (int)pb_line->line.len, pb_line->line.ptr, DATABASE_FILE_LIST);
+	return res;
+}
 
-	assert(value_str);
-	assert(value_str_size > 0);
-
-	/* Set default value */
-	*value_str = '\0';
-
-	while (*buf)
-	{
-		switch (state)
-		{
-			case CONTROL_WAIT_NAME:
-				if (*buf == '"')
-					state = CONTROL_INNAME;
-				else if (IsAlpha(*buf))
-					control_string_bad_format(str);
-				break;
-			case CONTROL_INNAME:
-				/* Found target field. Parse value. */
-				if (*buf == '"')
-					state = CONTROL_WAIT_COLON;
-				/* Check next field */
-				else if (*buf != *name_ptr)
-				{
-					name_ptr = (char *) name;
-					state = CONTROL_WAIT_NEXT_NAME;
-				}
-				else
-					name_ptr++;
-				break;
-			case CONTROL_WAIT_COLON:
-				if (*buf == ':')
-					state = CONTROL_WAIT_VALUE;
-				else if (!IsSpace(*buf))
-					control_string_bad_format(str);
-				break;
-			case CONTROL_WAIT_VALUE:
-				if (*buf == '"')
-				{
-					state = CONTROL_INVALUE;
-				}
-				else if (IsAlpha(*buf))
-					control_string_bad_format(str);
-				break;
-			case CONTROL_INVALUE:
-				/* Value was parsed, exit */
-				if (*buf == '"')
-				{
-					*value_str = '\0';
-					return true;
-				}
-				else
-				{
-					/* verify if value_str not exceeds value_str_size limits */
-					if (value_str - value_str_start >= value_str_size - 1) {
-						elog(ERROR, "field \"%s\" is out of range in the line %s of the file %s",
-							 name, str, DATABASE_FILE_LIST);
-					}
-					*value_str = *buf;
-					value_str++;
-				}
-				break;
-			case CONTROL_WAIT_NEXT_NAME:
-				if (*buf == ',')
-					state = CONTROL_WAIT_NAME;
-				break;
-			default:
-				/* Should not happen */
-				break;
-		}
-
-		buf++;
-	}
-
-	/* There is no close quotes */
-	if (state == CONTROL_INNAME || state == CONTROL_INVALUE)
-		control_string_bad_format(str);
-
-	/* Did not find target field */
-	if (is_mandatory)
-		elog(ERROR, "field \"%s\" is not found in the line %s of the file %s",
-			 name, str, DATABASE_FILE_LIST);
-	return false;
+int64_t
+pb_control_line_get_int64(pb_control_line *pb_line, const char *name)
+{
+	int64_t res;
+	if (!pb_control_line_try_int64(pb_line, name, &res))
+		elog(ERROR, "field \"%s\" is not found in the line %.*s of the file %s",
+			 name, (int)pb_line->line.len, pb_line->line.ptr, DATABASE_FILE_LIST);
+	return res;
 }
 
 static void
-control_string_bad_format(const char* str)
+control_string_bad_format(ft_bytes_t str)
 {
-    elog(ERROR, "%s file has invalid format in line %s",
-         DATABASE_FILE_LIST, str);
+	elog(ERROR, "%s file has invalid format in line %.*s",
+		 DATABASE_FILE_LIST, (int)str.len, str.ptr);
 }
 
 /*
@@ -1481,47 +1504,50 @@ write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_
 parray *
 read_database_map(pgBackup *backup)
 {
-	FILE		*fp;
 	parray 		*database_map;
-	char		buf[MAXPGPATH];
 	char		path[MAXPGPATH];
 	char		database_map_path[MAXPGPATH];
+	pioDrive_i	drive;
+	err_i 		err = $noerr();
+	ft_bytes_t	content;
+	ft_bytes_t	parse;
+	ft_bytes_t	line;
+	pb_control_line pb_line;
 
 	join_path_components(path, backup->root_dir, DATABASE_DIR);
 	join_path_components(database_map_path, path, DATABASE_MAP);
 
-	fp = fio_open_stream(FIO_BACKUP_HOST, database_map_path);
-	if (fp == NULL)
-	{
-		/* It is NOT ok for database_map to be missing at this point, so
-		 * we should error here.
-		 * It`s a job of the caller to error if database_map is not empty.
-		 */
-		elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno));
-	}
+	drive = pioDriveForLocation(FIO_BACKUP_HOST);
+
+	content = $i(pioReadFile, drive, .path = database_map_path, .binary = false,
+				 .err = &err);
+	if ($haserr(err))
+		ft_logerr(FT_FATAL, $errmsg(err), "Reading database_map");
 
 	database_map = parray_new();
 
-	while (fgets(buf, lengthof(buf), fp))
+	init_pb_control_line(&pb_line);
+
+	parse = content;
+	while (parse.len > 0)
 	{
-		char datname[MAXPGPATH];
+		ft_str_t datname;
 		int64 dbOid;
+		db_map_entry *db_entry = pgut_new0(db_map_entry);
 
-		db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry));
+		line = ft_bytes_shift_line(&parse);
+		parse_pb_control_line(&pb_line, line);
 
-        get_control_value_int64(buf, "dbOid", &dbOid, true);
-        get_control_value_str(buf, "datname", datname, sizeof(datname), true);
+        dbOid = pb_control_line_get_int64(&pb_line, "dbOid");
+        datname = pb_control_line_get_str(&pb_line, "datname");
 
 		db_entry->dbOid = dbOid;
-		db_entry->datname = pgut_strdup(datname);
+		db_entry->datname = ft_strdup(datname).ptr;
 
 		parray_append(database_map, db_entry);
 	}
 
-	if (ferror(fp))
-			elog(ERROR, "Failed to read from file: \"%s\"", database_map_path);
-
-	fio_close_stream(fp);
+	deinit_pb_control_line(&pb_line);
 
 	/* Return NULL if file is empty */
 	if (parray_num(database_map) == 0)

src/fu_util/ft_array.inc.h

@@ -163,6 +163,7 @@
 #define ft_array_ensure     fm_cat(ft_array_pref, _ensure)
 #define ft_array_recapa     fm_cat(ft_array_pref, _recapa)
 #define ft_array_resize     fm_cat(ft_array_pref, _resize)
+#define ft_array_reset_for_reuse     fm_cat(ft_array_pref, _reset_for_reuse)
 #define ft_array_free       fm_cat(ft_array_pref, _free)
 
 #define ft_array_insert_at  fm_cat(ft_array_pref, _insert_at)
@@ -373,6 +374,11 @@ ft_array_resize(ft_array_type *arr, size_t len) {
         ft_array_recapa(arr, arr->len);
 }
 
+ft_inline void
+ft_array_reset_for_reuse(ft_array_type *arr) {
+	arr->len = 0;
+}
+
 ft_inline ft_array_type
 ft_array_alloc(FT_SLICE_TYPE *ptr, size_t len) {
     ft_array_type   arr = {NULL, 0, 0};
@@ -576,6 +582,7 @@ ft_array_walk(ft_array_type *arr, FT_WALK_ACT (*walk)(FT_SLICE_TYPE *el)) {
 #undef ft_array_ensure
 #undef ft_array_recapa
 #undef ft_array_resize
+#undef ft_array_reset_for_reuse
 #undef ft_array_free
 
 #undef ft_array_insert_at

src/pg_probackup.h

@@ -970,9 +970,20 @@ extern CompressAlg parse_compress_alg(const char *arg);
 extern const char* deparse_compress_alg(int alg);
 
 /* in dir.c */
-extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory);
-extern bool get_control_value_str(const char *str, const char *name,
-                                  char *value_str, size_t value_str_size, bool is_mandatory);
+typedef struct ft_arr_clkv_t ft_arr_clkv_t;
+typedef struct pb_control_line pb_control_line;
+struct pb_control_line {
+	ft_bytes_t     line;
+	ft_arr_clkv_t *kvs;
+	ft_strbuf_t    strbuf;
+};
+extern void init_pb_control_line(pb_control_line *pb_line);
+extern void parse_pb_control_line(pb_control_line *pb_line, ft_bytes_t line);
+extern void deinit_pb_control_line(pb_control_line *pb_line);
+extern int64_t pb_control_line_get_int64(pb_control_line *pb_line, const char *name);
+extern ft_str_t pb_control_line_get_str(pb_control_line *pb_line, const char *name);
+extern bool pb_control_line_try_int64(pb_control_line *pb_line, const char *name, int64 *value);
+extern bool pb_control_line_try_str(pb_control_line *pb_line, const char *name, ft_str_t *value);
 
 extern const char *get_tablespace_mapping(const char *dir);
 extern void create_data_directories(parray *dest_files,

src/utils/configuration.c

@@ -359,8 +359,6 @@ assign_option(ConfigOption *opt, const char *optarg, OptionSource src)
 	}
 }
 
-#define SPACES " \t\n\v\f\r"
-
 static bool
 get_next_token(ft_bytes_t *src, ft_strbuf_t *dest)
 {

src/utils/pgut.h

@@ -88,6 +88,8 @@ extern void  pgut_free(void *p);
 #define AssertMacro(x)	((void) 0)
 #endif
 
+#define SPACES " \t\n\v\f\r"
+
 #define IsSpace(c)		(isspace((unsigned char)(c)))
 #define IsAlpha(c)		(isalpha((unsigned char)(c)))
 #define IsAlnum(c)		(isalnum((unsigned char)(c)))