[ { "commit": "fd8974914d8702e5d889f03e08eeda676435faa2", "date": "2024-01-12 13:55:08 -0300", "subject": "Cleanup tablespace tests in command/backup module.", "body": "Tablespaces were enabled for tests that did not have tablespaces, resulting in tablespace_map being present in backups even when it was not needed.\n\nInstead of specifying if tablespaces are present, automatically detect tablespaces in hrnBackupPqScript()." }, { "commit": "3926dd346ef7b6e239e939e9a046a4be8597c584", "date": "2024-01-04 14:55:44 -0300", "subject": "Update LICENSE.txt and PostgreSQL copyright for 2024." }, { "commit": "a484862763e85a7cc11b6a45366b3b78b06cd373", "date": "2024-01-04 14:53:44 -0300", "subject": "Remove installation of old yum.p.o repository key." }, { "commit": "55f22489630ae409f03a2ad9ec37c3e38a3f7ab2", "date": "2024-01-03 18:15:50 -0300", "subject": "Break up scripts in db unit test module.", "body": "Long scripts followed by a number of tests are really challenging to debug and update.\n\nInstead, break up the scripts to be inline with the tests that they drive. This should make maintenance of the tests much simpler." }, { "commit": "7c17eec3db455d3e8e660f4f8247b6975517648f", "date": "2024-01-03 13:01:50 -0300", "subject": "Improve SFTP storage error messages.", "body": "Use storageSftpEvalLibSsh2Error() in more locations to provide better error information. Also add storageSftpLibSsh2SessionLastError() for the same reason." }, { "commit": "802ae7914897f3d53788e526ce22562e434c92ea", "date": "2024-01-03 12:43:50 -0300", "subject": "Remove FreeBSD 12 and add FreeBSD 14 to Cirrus CI.", "body": "FreeBSD 12 is now EOL.\n\nAlso update the image version for FreeBSD 13." }, { "commit": "8a8cfba62bfe2293d6355f68d44bab5323a91118", "date": "2023-12-27 12:53:53 -0300", "subject": "Remove resolved comment." }, { "commit": "f4a4af299bee93ab7dde599ccdb35becc1c2afad", "date": "2023-12-27 12:49:47 -0300", "subject": "Fix short read in block incremental restore.", "body": "During restore it is possible to read all the blocks out of a compressed super block without reading all the input. This is because the compression format may have some trailing bytes that are not required for decompression but are required to indicate that data has ended. If a buffer aligned with the compressed data in a certain way, these last bytes might not be read.\r\n\r\nExplicitly read out any final bytes at the end of each super block to handle this case. This should always result in no additional data out and we check for that, but it does move the read position to the beginning of the next compressed super block so decompression can begin without error." }, { "commit": "c47b39acf67b375cbee9773904ad1afc6a1bb9ec", "date": "2023-12-27 12:39:45 -0300", "subject": "Fix incorrect test comment.", "body": "Left over from an older implementation." }, { "commit": "4324b568a922105b272e2031559dae5f374a04da", "date": "2023-12-26 21:07:56 -0300", "subject": "Move block testBlockDelta() to harness module.", "body": "This makes the function available to other test modules.\n\nAlso rename to hrnBlockDeltaRender()." }, { "commit": "9049fec2c02c4110e16ffdff7b56f83d8ff32b4a", "date": "2023-12-22 13:16:45 -0300", "subject": "Refactor skip files truncated during backup when bundling.", "body": "Refactor 02eea555 to always close the file immediately on EOF and use backupCopyResultCopy to continue processing. Closing the file immediately saves a later EOF check and is friendlier to added logic in this area. Using backupCopyResultCopy to continue is clearer also makes it easier to add new logic.\n\nAlso store zero checksum so the bulk of results collection can be moved within the copy block." }, { "commit": "c8795094d418802a0e24d4d19da66973c71b00e5", "date": "2023-12-22 12:48:01 -0300", "subject": "Allow const checksum buffers to be returned from backupFile().", "body": "This allows less duplication of buffers.\n\nFor delta check return file->pgFileSize/file->pgFileChecksum instead of pgTestSize/pgTestChecksum since this saves one buffer duplication and we know these values are equal since we just checked them.\n\nAlso add an assert to ensure copyChecksum is valid relative to size." }, { "commit": "4f760df417e17dc03ce07c0116dfaac506e04c84", "date": "2023-12-22 00:59:12 -0300", "subject": "Do not preserve block incremental if file is less than prior block size.", "body": "If a file stored with block incremental shrinks below the prior block size then the map is useless and the entire file needs to be stored again.\r\n\r\nIn this case use the new block incremental values (even if none) rather than preserving the old ones." }, { "commit": "3cd8249dbaa085bd075464d420f68a2a8b34a165", "date": "2023-12-21 15:28:37 -0300", "subject": "Update warning for backup resume invalid repo file.", "body": "This warning has had a note since the C migration that it should be moved below the backup file log message, so do that.\n\nAlso update the warning message a bit to correct for tense. This message was likely in a different place originally." }, { "commit": "701865eca1c8896a913e05f41a748c225b84af11", "date": "2023-12-21 15:20:03 -0300", "subject": "Refactor backupFile() to remove backupCopyResultReCopy.", "body": "Having two enum values for file copy makes things a bit more complicated than they need to be (especially in an upcoming commit).\n\nInstead add a flag to indicate that the repository file was invalid since the only purpose is to trigger a warning message." }, { "commit": "a42614e8f3a9e6871c16331f01c2cd765ab181aa", "date": "2023-12-21 15:08:07 -0300", "subject": "Preserve partial files during block incremental delta restore.", "body": "Previously files that were smaller than the expected size were not preserved for block incremental, even though it is possible that block incremental could make use of a partial file.\r\n\r\nOne example is when a restore encounters an error. On retry the partial file can be used as a starting point rather than copying again from the beginning. Another example is restoring a backup where a file is larger than what already exists in the data directory.\r\n\r\nPreserve any size file when block incremental will be used for the delta in order to reuse partial files when possible. If the file is smaller than expected then disable the whole-file checksum to reduce overhead." }, { "commit": "ad8febec0851473240f6f20bd4eb299b525a6d44", "date": "2023-12-21 13:32:10 -0300", "subject": "Refactor backup incremental manifest generation.", "body": "This refactor should provide more clarity on what factors affect an incremental, rather that just having one big expression do it all. Overall this may be slightly more efficient since some values are reused that before were recalculated.\r\n\r\nNo behavioral changes are introduced." }, { "commit": "f3584e2143de518bda52151a7102ed638dcb0e95", "date": "2023-12-21 11:20:10 -0300", "subject": "Add tests to command/backup and info/manifest modules.", "body": "These tests exercise various interesting cases and provide coverage for proposed improvements." }, { "commit": "25f14898babf4ad78baef3e8343933a863633975", "date": "2023-12-21 10:16:13 -0300", "subject": "Fix overflow suppressing backup progress in info output.", "body": "Writing the sz and szCplt parameters in the lock file used jsonWriteUInt64() but reading these parameters used jsonReadUInt(). This caused a silent exception for any backups larger than MAX_UINT and prevented the info command from reporting progress.\r\n\r\nCorrect this so the reads are symmetric and verified before/after with a test." }, { "commit": "8af3c1c9acc7117a0cf8e3ce862c5639c82bf78d", "date": "2023-12-17 13:15:03 -0300", "subject": "Use original file size to log size changes during backup.", "body": "c9703b35 added logging for file size changes during backup. Later 5ed6f8df added the sizeOriginal member to ManifestFile, which arguably is better to use for logging rather than size before backup since it will always contain the original size. Size could in theory be modified for deduplication purposes.\n\nUpdate logging to use sizeOriginal." }, { "commit": "bb6e5164ee6efd40fde73acd8f7b30c4afa1548d", "date": "2023-12-16 11:42:27 -0300", "subject": "Add block incremental test where timestamp changes but file is the same.", "body": "If delta is not enabled, then the timestamp is used to determine if a file has changed. If the timestamp changes but the file is the same then the prior map will be stored unchanged in the new backup. This is not quite as bad as storing the entire file but it is obviously not ideal.\n\nThis will be fixed in a future commit, but add the test now to show the current behavior." }, { "commit": "02eea555c7169e99892fdbc8ce7cc1ee59f50509", "date": "2023-12-14 14:21:06 -0300", "subject": "Skip files truncated during backup when bundling.", "body": "In bundle mode pgBackRest skips files of zero size, that is, it does not queue them for copying.\r\n\r\nAfter splitting the files into bundles, pgBackRest launches one or more processes that directly perform the backup, namely, read the files and, if necessary, write them to the bundles.\r\n\r\nIf during the time between the distribution of all files among bundles and the direct copying of a file to a bundle, this file of non-zero size was truncated to zero size (for example, when the table was truncated), then pgBackRest still unconditionally places such a zero-size file in the bundle, taking up space in it equal to the size of the headings, and additionally writes the original file size to the manifest.\r\n\r\nIn debug build an assertion was added, that does not allow zero-size files to be written to bundles, which leads to an error.\r\n\r\nTo solve the problem, this patch, when reading the next file, loads one buffer from the file to detect if it is zero-size. If so it marks the file as truncated and continues on to the next file.\r\n\r\nThe advantages of the solution are that, firstly, the assert will not fire on debug builds, and secondly, we will not place zero-size files in bundles, which exactly corresponds to the specification.\r\n\r\nThe patch adds the backupCopyResultTruncate value to the BackupCopyResult enumeration to use it to indicate the result when a non-zero size file is truncated to zero size during the backup process." }, { "commit": "89d5278b74ab0d880345575d7ac4c239cd1fce54", "date": "2023-12-14 13:28:52 -0300", "subject": "Add support for alternate compile-time page sizes.", "body": "Alternate pages sizes can be selected at compile-time, .e.g. 4096. While compile-time settings are generally not well tested by core, some established forks such as Greenplum use them." }, { "commit": "d205a61949ae8491f5ce9b2019cda79d2fe7b216", "date": "2023-12-01 11:54:30 -0300", "subject": "Fix flapping test on older ninja versions in test unit.", "body": "Older versions of ninja may fail to rebuild correctly when changes are made to the configuration. In this case there is an automatic retry but the unexpected log output would cause the test to fail.\n\nFor tests that are expected to succeed, check that the log is empty but also accept a retry message as long as the test does eventually succeed.\n\nAdd a new harness function, harnessLogResultEmptyOrContains(), to make this work and also clean up some adjacent code." }, { "commit": "7ce0f5a94c229abe415029ce80241a64665c93a2", "date": "2023-11-30 16:43:09 -0300", "subject": "Use unique port for each server unit test.", "body": "If the same port is reused too quickly bind may fail with this error:\n\nFileOpenError: unable to bind socket: [98] Address already in use\n\nWe specify SO_REUSEADDR when creating the socket but apparently this is not always enough if the port is reused very rapidly.\n\nFix this (hopefully) by using a unique port for each test that needs one. This does in theory limit the number of tests that can use ports, but we allow 768 per test, whereas the test that uses the most ports is common/io-tls with 4." }, { "commit": "a14732789baacb4d35d5da6d0adf337ceea7f595", "date": "2023-11-29 09:31:57 -0300", "subject": "Output coverage report on test failure in CI.", "body": "This allows analysis of coverage failures that only happen in CI. It is not ideal since the report needs to be copied from the log output into an HTML file where it can be viewed, but better than nothing." }, { "commit": "cb6bceb9f1074436b7d0b67a94d782113b5f2b35", "date": "2023-11-28 16:38:42 -0300", "subject": "Improve comments in socket test harness." }, { "commit": "70e15dacc77bcc2b7e4c2a222d683872d2180f5b", "date": "2023-11-28 16:35:37 -0300", "subject": "Allow custom type/message for errRetryAdd().", "body": "It may be useful to customize the message or add a message that was never thrown. The latter case will be used in an upcoming commit." }, { "commit": "85bc9f27d8b70a879051f604861078034263b095", "date": "2023-11-27 09:06:53 -0300", "subject": "Begin v2.50 development." }, { "commit": "3cb891e3ca49ed1166fb0b513b126412448f5b09", "date": "2023-11-27 08:55:56 -0300", "subject": "v2.49: Remove PostgreSQL 9.3 Support" }, { "commit": "7d51228bf5227494bf50327488032221bd3d30fa", "date": "2023-11-24 17:07:49 -0300", "subject": "Migrate backupFile() tests in command/backup module.", "body": "The backupFile() tests were written before the bulk of the backup command had been migrated to C. Some of them have been migrated to the complete backup tests, but others were left because there was no way to make changes to files during a backup.\n\nNow that we have the backup script harness introduced in 337da35a it is now possible to migrate all the tests. The new tests are better because they not only test backupFile() but all the functions upstream and downstream of it." }, { "commit": "337da35ab278e058ec4cdffeb4319f1a1e2cb0df", "date": "2023-11-24 12:25:40 -0300", "subject": "Add test to show behavior of bundled files truncated during backup.", "body": "This behavior violates an assertion but is completely possible with the current implementation. This behavior will be fixed in a future commit, but for now at least test how it works correctly and remove the assertion so the test runs without error.\n\nAlso add a new harness that allows changes during the backup to be scripted." }, { "commit": "ac78b965837440ee34f7b5bf0a039c6e9d78eeec", "date": "2023-11-18 11:12:41 -0300", "subject": "Remove unused fields from backupJobResult() test in command/backup unit.", "body": "These fields were not used because of the noop so it was hard to keep them up to date. Rather than attempt to do so, just remove them and add a comment to explain why they are missing." }, { "commit": "c4dc4665f88e9d67a3b66f44cd59d6dd4bb5107f", "date": "2023-11-18 10:47:58 -0300", "subject": "Fix recursive path remove in SFTP storage driver.", "body": "storageSftpPathRemove() used LIBSSH2_FX_FAILURE to determine when it was attempting to unlink a directory, but it appears that LIBSSH2_FX_PERMISSION_DENIED is also valid for this case.\r\n\r\nUpdate storageSftpPathRemove() to accept either error and adjust tests." }, { "commit": "e2b734eff97e3fac32895b799cc32126013aae27", "date": "2023-11-16 12:50:09 -0300", "subject": "Allow config/load unit test to run without libssh2 installed.", "body": "Add additional #ifdef HAVE_LIBSSH2 wrapping around tests requiring libssh2 in loadTest.c." }, { "commit": "05207bb8e4f462cd2fd5c685d43ad8911b219077", "date": "2023-11-15 09:41:40 -0300", "subject": "Fix storageReadRemote() to return actual read bytes.", "body": "All storage interface read methods should return actual read bytes. This patch refactors storageReadRemote() to eliminate duplicated code and return actual read bytes. The return value is calculated as the number of bytes written to the passed buffer.\r\n\r\nThis is technically a bug but does not express as an issue currently because this return value is not being used. It will be used in the future, though, so it needs to be fixed." }, { "commit": "ea317df5d9af66935175f7ec87845f30be4f5d36", "date": "2023-11-11 10:33:09 -0300", "subject": "Remove old version conditionals from user guide.", "body": "The user guide does not need to build for EOL versions of PostgreSQL, so remove some conditionals used to support versions older than 12." }, { "commit": "eb69e2ee6339047f0145b751437ac2bec82bb5ab", "date": "2023-11-10 17:00:57 -0300", "subject": "Update point-in-time recovery documentation for PostgreSQL >= 13.", "body": "PITR changed in PostgreSQL 13 to error when the recovery target is not reached. Update the documentation to work with PostgreSQL >= 13 as well as < 13.\n\nAlso update the versions built for RHEL and Debian since PostgreSQL 11 is now EOL." }, { "commit": "dcf078198761a2cfe9be6b4c7f4b4fb07ef53cbf", "date": "2023-11-09 12:59:12 -0300", "subject": "Remove support for PostgreSQL 9.3.", "body": "Per our policy to support five EOL versions of PostgreSQL, 9.3 is no longer supported by pgBackRest.\r\n\r\nRemove all logic associated with 9.3 and update the tests." }, { "commit": "fa5b2d44ad9f7591517eecaaaaeb07e9500dd51c", "date": "2023-11-09 12:04:25 -0300", "subject": "Fix regression in retries.", "body": "5314dbf aimed to make nested Wait objects more accurate with regard to wait time but it also got rid of the \"bonus\" retry that was implicit in the prior implementation. This meant that if an operation used up the entire allotted timeout, it would not be retried. Object stores especially are noisy places and some amount of retry should always be attempted. So even though removing the \"bonus\" retry was intended, it turned out not to be a good idea.\r\n\r\nInstead of an implicit retry, formalize two retries in the Wait object even if the wait time has expired. Any number of retries are allowed during the wait period. Also remove waitRemaining() since it is no longer needed.\r\n\r\nAdjust tests as needed to account for the extra timeouts.\r\n\r\nNote that there may still be an underlying issue here that is simply being masked by retries. That is, the issue expressing was that waiting for a socket to be writable was timing out and without a retry that caused a hard error. This patch does nothing to address the source of the write timeout and perhaps there is nothing we can do about it. It does seem similar to the write issue we had with our blocking TLS implementation, but it was never clear if that was a problem with TLS, the kernel, or a bug in pgBackRest itself. It cropped up after a kernel update and we switched to non-blocking TLS to address the issue (c88684e)." }, { "commit": "3c116e1829a0df62aaf50ce241c63d829eb438e0", "date": "2023-11-04 14:43:11 -0300", "subject": "Remove unused header." }, { "commit": "e0f5880b09624db2c9274a8a615a5309a0bddab1", "date": "2023-10-22 13:55:56 -0400", "subject": "Refactor of pq shim to allow more flexible scripting.", "body": "The pq scripts were pretty static which had already led to a lot of code duplication in the backup test harness.\n\nInstead allow the scripts to be built dynamically, which allows for much more flexibility and reduces duplication. For now just make these changes in the backup harness, but they may be useful elsewhere.\n\nWhile we are making big changes, also update the macro/function names to hew closer to our current harness naming conventions." }, { "commit": "306fdff93a072c72d6a40aab728ceeb44cc7b23f", "date": "2023-10-19 10:11:27 -0400", "subject": "Remove unused parameter in backupProcess().", "body": "Also fix a comment in the same function." }, { "commit": "04d92cca7eb42e0037e65701109da284b214c84f", "date": "2023-10-19 09:54:34 -0400", "subject": "Tidy and align site introduction and description.", "body": "Make the description more concise and use it for both the site description and introduction." }, { "commit": "459d59615a18c147140b1ae94a804be66541c0ab", "date": "2023-10-18 18:14:32 -0400", "subject": "More efficient/compact calculations for encoded sizes.", "body": "encodeToStrSizeBase64() is definitely more efficient (pulled from the PostgreSQL implementation).\n\nencodeToStrSizeBase64Url() is probably about as efficient as the prior implementation but is certainly more compact.\n\nAlso add tests for zero byte encoding sizes." }, { "commit": "21c8c8a66cb06eba714bfa41bed7239af3d14f08", "date": "2023-10-14 16:22:09 -0400", "subject": "Document maintainer options.", "body": "Document maintainer options in a separate section with appropriate explanation and caveats.\r\n\r\nAlso make the pg-version-force option user visible now that maintainer caveats have been documented." }, { "commit": "81536cd486d716ebd10eb0637ef6b1b9a61d9dd0", "date": "2023-10-10 16:49:05 -0400", "subject": "Simplify description rendering in command and configuration reference.", "body": "The reference documentation was still using a very old version of rendering from before the user guide was introduced. This was preserved in the initial C migration to reduce the diff between Perl and C for testing purposes. The old version used hard linefeeds to simulate paragraphs and reduce the amount of markup that needed to be used. In retrospect this was not a great idea.\n\nInstead use more natural rendering that does not depend on using hard linefeeds between paragraphs." }, { "commit": "45abea471ecec10a351bfd9e5fbed8737c32da42", "date": "2023-10-10 13:02:50 -0400", "subject": "Simplify section titles in configuration reference.", "body": "For some reason the internal section id was included in the title. This was probably copied from another section title where it made more sense, e.g. including the option name after the title.\n\nAlso add release note missed in 1eb01622." }, { "commit": "1eb0162208b37f577d35ba775f007202b285900a", "date": "2023-10-09 14:03:43 -0400", "subject": "Build command and configuration reference in C.", "body": "Migrate generation of these files from help.xml to the intermediate documentation format. This allows us to share a lot of code that is already in C and remove duplicated code in Perl. More duplicate code can be removed in Perl once man generation is migrated.\n\nAlso update the unit test harness to allow testing of modules in the doc directory." }, { "commit": "983cc1a9e3f2865bf5128b5065722fd995382ba1", "date": "2023-10-01 16:32:27 -0400", "subject": "Adjust timeouts in the common/io/tls test to fix flapping coverage.", "body": "This test was failing coverage pretty regularly because the retry in tlsClientOpen() was not always being reached. Make the TLS timeouts longer to ensure reliable coverage." }, { "commit": "af4621894a20ca0333726d459e2e6409bd4ee1e0", "date": "2023-09-30 12:47:15 -0400", "subject": "Fix common/lock test failing with -Werror=unused-result.", "body": "Wrap ftruncate() in TEST_RESULT_INT() to prevent this error." }, { "commit": "eccd9eed196ff9adc074061f26fec523eae2ba00", "date": "2023-09-30 12:45:24 -0400", "subject": "Update Minio test/documentation container version." }, { "commit": "33ba4db9cbd334c9a10f15880e13e3fb589c3b10", "date": "2023-09-30 09:40:44 -0400", "subject": "Use CSS to number sections in documentation.", "body": "This reduces churn in the HTML when sections are added or removed from the documentation." }, { "commit": "1d5563288cf1839acb237a188b865ecb01fe97da", "date": "2023-09-29 17:28:00 -0400", "subject": "Parse defaults and text sections in help.xml.", "body": "These will be required to build documentation in C." }, { "commit": "088026e6ffcd1b084099e11f8461fdd3895b2507", "date": "2023-09-29 17:06:37 -0400", "subject": "Allow documentation source file to be specified in manifest.", "body": "The help source file had previously been hardcoded and now that is no longer needed.\n\nA future commit will introduce more sources outside of the xml path." }, { "commit": "55fda01733f2a7e71ee175345e26344a99989927", "date": "2023-09-29 16:57:01 -0400", "subject": "Remove unused references to DocConfig and DocConfigData Perl modules." }, { "commit": "217584a2c4134667e82b2de8032b5334214e08f1", "date": "2023-09-29 16:52:08 -0400", "subject": "Add new XML functions required for building documentation." }, { "commit": "8f319b6fd34f3430b5b527825bd502113f8a957d", "date": "2023-09-25 09:43:30 -0400", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "084c8e1691f59aa412b859f9004e188383826c58", "date": "2023-09-25 09:40:45 -0400", "subject": "Begin v2.49 development." }, { "commit": "a7ab686d0e3d48b3200986d91ea3489488800382", "date": "2023-09-25 09:32:15 -0400", "subject": "v2.48: Repository Storage Tags" }, { "commit": "cb3ff6ed4336af9e4c0643d8224d5bc3d790a655", "date": "2023-09-23 13:41:03 -0400", "subject": "Fix command reference example for the filter option.", "body": "This example was broken by 24f7252. Revert to (almost) the prior code to fix this example until something better can be committed. The something better is in progress but it adds new build requirements so it is too late to include it for the release.\n\nTechnically this breaks some other examples, but they are all internal and not visible in the user-facing documentation." }, { "commit": "6f0f2b371e6604552001b56e8931c6d15ceab41f", "date": "2023-09-19 19:01:16 -0400", "subject": "Update help title for the --annotation option.", "body": "Fix capitalization and remove pluralization that is implied." }, { "commit": "c3c0834e170196a1241a368779162fcca65f9ad5", "date": "2023-09-19 16:26:13 -0400", "subject": "Allow storage/sftp unit test to run without libssh2 installed.", "body": "Add missing #ifdefs and update tests to prevent test compilation failure and test run issues when libssh2 is not present." }, { "commit": "bb752cd111b1257088e57137bdbe6715e96eb292", "date": "2023-09-19 12:08:09 -0400", "subject": "Remove duplicate tests from storage/sftp unit test module.", "body": "These tests are already run as part of storage/posix and do not need to be duplicated in storage/sftp." }, { "commit": "31de127cf42438b1fac04c161c47264aa1fb1060", "date": "2023-09-19 11:30:29 -0400", "subject": "Fix issue restoring block incremental without a block list.", "body": "It is currently possible for a block map to be written without any accompanying blocks. This happens when a file timestamp is updated but the file has not changed. On restore, this caused problems when encryption was enabled, the block map was bundled after a file that had been stored without block incremental, and both files were included in a single bundle read. In this case the block map would not be decrypted and the encrypted data was passed to blockMapNewRead() with unpredictable results. In many cases built-in retries would rectify this problem as long as delta was enabled since block maps would move to the beginning of the bundle read and be decrypted properly. If enough files were affected, however, it could overwhelm the retries and throw an error. Subsequent delta restores would eventually be able to produce a valid result.\r\n\r\nFix this by moving block map decryption so it works correctly no matter where the block map is located in the read. This has the additional benefit of limiting how far the block map can read so it will error earlier if corrupt. Though in this case there was no repository corruption involved, it appeared that way to blockMapNewRead() since it was reading encrypted data.\r\n\r\nArguably block maps without blocks should not be written at all, but it would be better to consider that as a separate change. This pattern clearly exists in the wild and needs to be handled, plus the new implementation has other benefits." }, { "commit": "88edea4571d954e54a561247de587f747eaaadca", "date": "2023-09-18 11:30:42 -0400", "subject": "Add block incremental info to restore detail logging.", "body": "Log that block incremental was used and the delta size if less than the entire file was updated." }, { "commit": "d27533b40f039ff8da3a284d77fccff101083492", "date": "2023-09-17 17:06:42 -0400", "subject": "Fix manifest command filter for files in a tablespace.", "body": "pg_data/ was appended at the beginning of the filter, which meant that files in tablespaces could never be queried directly.\n\nUpdate the filter to require the full path, including pg_data/ or pg_tblspc/." }, { "commit": "ed88f0483e8c87fde6dc098102b1c49e281d96e1", "date": "2023-09-16 09:35:27 -0400", "subject": "Add missing comma in protocolParallelJobToLog()." }, { "commit": "ce9ba0fadee652d409488bee701ea6a51cafec75", "date": "2023-09-15 20:22:38 -0400", "subject": "Add known hosts checking for SFTP storage driver.", "body": "By default require a known hosts match as part of the SFTP storage driver's authentication process, i.e. repo-sftp-host-key-check-type=strict. The match is expected to be found in the default list or in a list of known hosts files provided by the user. An exception is made if a fingerprint has been manually configured with repo-sftp-host-fingerprint or repo-sftp-host-key-check-type=accept-new can be used to automatically add new hosts.\r\n\r\nAlso allow host key verification to be skipped, as before, but require the user to explicitly set this (repo-sftp-host-key-check-type=none) rather than it being the default." }, { "commit": "f5c730fd03ff27088bd91c148ed285c63b5e7502", "date": "2023-09-15 18:15:02 -0400", "subject": "Add prefix to meson build target to avoid conflicts.", "body": "Older versions of meson fail when a build target in a subproject has the same name as another subproject.\n\nThis has been fixed in newer versions, but we still need to support older versions and in any case this seems cleaner and the help build target is already prefixed in this fashion." }, { "commit": "9039d20b5bd73d3732ed6dac0a2bba4558176d8f", "date": "2023-09-15 09:30:40 -0400", "subject": "Add report option to check command.", "body": "This option is intended to eventually create a comprehensive report about the state of the pgBackRest configuration based on the results of the check command.\r\n\r\nImplement a detailed report of the configuration options in the environment and configuration files. This should be useful information when debugging configuration errors, since invalid options and configurations are automatically noted. While custom config locations will not be found automatically, it will at least be clear that the config is not in a standard location.\r\n\r\nFor now keep this option internal since there is a lot of work to be done, but commit it so that it can be used when needed and tested in various environments.\r\n\r\nNote that for now when --report is specified, the check command is not being run at all. Only the config report is generated. This behavior will be improved in the future." }, { "commit": "657c1a3e069910106bc3642153c27b0542c61dc3", "date": "2023-09-14 18:41:36 -0400", "subject": "Finalize catalog number for PostgreSQL 16 release." }, { "commit": "1b4e0cce5f3ff5ea3ee0b889e6d4ce4de1430b04", "date": "2023-09-14 08:22:21 -0400", "subject": "Add --repo-storage-tag option to create object tags.", "body": "This new option allows tags to be added to objects in S3, GCS, and Azure repositories.\r\n\r\nThis was fairly straightforward for S3 and Azure, but GCS does not allow tags for a simple upload using the JSON interface. If tags are required then the resumable interface must be used even if the file falls below the limit that usually triggers a resumable upload (i.e. size < repo-storage-upload-chunk-size).\r\n\r\nThis option is structured so that tags must be specified per-repo rather than globally for all repos. This seems logical since the tag keys and values may vary by service, e.g. S3 vs GCS.\r\n\r\nThese storage tags are independent of backup annotations since they are likely to be used for different purposes, e.g. billing, while the backup annotations are primarily intended for monitoring." }, { "commit": "3b9c31f6e3a0c23674f97b02c42133192fb51724", "date": "2023-09-13 16:59:52 -0400", "subject": "Use PROJECT_NAME constant instead of string literal." }, { "commit": "39bb8a0d3a96eef63219dff262a9306a4f59dae9", "date": "2023-09-12 18:09:58 -0400", "subject": "Support for dual stack connections.", "body": "The prior code would only connect to the first address provided by getaddrinfo().\r\n\r\nInstead try each address in the list. If all connections fail then wait and try them all again until timeout.\r\n\r\nCurrently a round robin approach is used where each connection attempt must fail before the next connection is attempted. This works fine, for example, when an ipv6 address has no route to the host, but will work less well when a host answers but doesn't respond in a timely fashion.\r\n\r\nWe may consider a Happy Eyeballs approach in the future, but since pgBackRest is primarily a background process, it is not clear that slightly improved response time (in the case of failed connections) is worth the extra complexity." }, { "commit": "9d3a605900644059ff0e030d71a818b3965eb031", "date": "2023-09-10 12:45:58 -0400", "subject": "Optimize WAL segment check after successful backup.", "body": "The prior code did one list command against the storage for each WAL segment. This led to a lot of lists and was especially inefficient when the WAL (or the majority of it) was already present.\r\n\r\nOptimize to keep the contents of a WAL directory and use them on a subsequent search. Leave the optimizations for a single WAL segment since other places still use that mode." }, { "commit": "edbd520c81f5d437963aaf42d1c79b2c51d01665", "date": "2023-09-10 10:36:58 -0400", "subject": "Refactor address list lookup to include all returned addresses.", "body": "sckHostLookup() only returned the first address record returned from getaddrinfo(). The new AddressInfo object provides a full list of values returned from getaddrinfo(). Freeing the list is also handled by the object so there is no longer a need for FINALLY blocks to ensure the list is freed.\r\n\r\nAdd the selected address to the client/server names for debugging purposes.\r\n\r\nThis code does not attempt to connect to multiple addresses. It just lays the groundwork for a future commit to do so." }, { "commit": "f42d927d2dcd58cf5a80d68eaf6888bcf2c67a4b", "date": "2023-09-10 09:47:49 -0400", "subject": "Retry reads of pg_control until checksum is valid.", "body": "On certain file systems (e.g. ext4) pg_control may appear torn if there is a concurrent write while reading the file. To prevent an invalid read, retry until the checksum matches the control data.\r\n\r\nSpecial handling is required for the pg-version-force feature since the offset of the checksum is not known. In this case, scan from the default position to the end of the data looking for a checksum match. This is a bit imprecise, but better than nothing, and the chance of a random collision in the control data seems very remote considering the ratio of data size (< 512 bytes) to checksum size (4 bytes).\r\n\r\nThis was discovered and a possible solution proposed for PostgreSQL in [1]. The proposed solution may work for backup, but pgBackRest needs to be able to read pg_control reliably outside of backup. So no matter what fix is adopted for PostgreSQL, pgBackRest need retries. Further adjustment may be required as the PostgreSQL fix evolves.\r\n\r\n[1] https://www.postgresql.org/message-id/20221123014224.xisi44byq3cf5psi%40awork3.anarazel.de" }, { "commit": "c1805134b370f0c5c54c40a2df8978599328784f", "date": "2023-09-09 12:54:55 -0400", "subject": "Aggregate error retries in ErrorRetry output.", "body": "If there are a lot of retries then the output might be very large and even be truncated by the error module. Either way, it is not good information for the user.\n\nWhen a message is repeated, aggregate so that total retries and time range are output for the message. This provides helpful information about what happened without overwhelming the user with data." }, { "commit": "5314dbffc761bc7507db02693f2421c1578efe10", "date": "2023-09-09 11:22:33 -0400", "subject": "Adjust Wait object to be more accurate when nested.", "body": "The prior code gave a \"free\" extra iteration at the end of the wait, functionality that was copied directly from the equivalent code in Perl. This works and is mostly negligible except when wait loops are nested, in which case outer loops will always run twice even if an inner loop times out, which has a multiplying effect. For example, three nested wait loops with a timeout of three seconds will result in the inner loop being run four times (for a total of twelve seconds) even if it times out each time.\n\nInstead make waitMore() stop exactly when time is up. This makes more sense because a complete failure and timeout of an inner loop means retrying an outer loop is probably a waste of time since that inner loop will likely continue to fail.\n\nAlso make waitRemaining() recalculate the remaining time rather than depending on the prior result.\n\nSome tests needed to be adjusted to take into account there being one less loop. In general this led to a simplification of the tests.\n\nReinit a begin value in the wait unit tests. This is not related to the current change but it does make the time measurements more accurate and less likely to fail on an edge case, which has been observed from time to time.\n\nThis change appears to have a benefit for test runtime, which seems plausible especially for nested waits, but a larger sample of CI runs are needed to be sure." }, { "commit": "d57900839addeac56b89a445352a8efc96a197f0", "date": "2023-09-04 18:51:44 -0400", "subject": "Improve backup test harness to handle very large quantities of WAL.", "body": "The current tests only generate small quantities of WAL per backup but sometimes it is useful to generate large quantities for testing.\n\nFix the issues with generating large quantities of WAL and also improve memory management." }, { "commit": "43524a47a4c6a847f76514812f94b87cb7af4c05", "date": "2023-09-04 13:39:11 -0400", "subject": "Fix configuration reference example for the tls-server-address option.", "body": "The prior example (::*) was not valid and would result in the following error:\r\n\r\nERROR: [049]: unable to get address for '::*': [-2] Name or service not known\r\n\r\nCorrect values are either * for IPv4 or :: for IPv6. The IPv4 value is used as the example since only one example is allowed." }, { "commit": "c0935b79924686ebf4efef99c7dc69aea9efb28d", "date": "2023-09-04 12:00:06 -0400", "subject": "Split release notes into individual files.", "body": "The release.xml file was getting pretty unwieldy so break release notes into separate files. Also break contributors into a separate file.\n\nIn theory most of release.xml could now be generated automatically but adding a new release does not represent a serious maintenance burden, so for the time being it does not seem worth it." }, { "commit": "6cb9c40fb8ea18e6572bf3f6e44bbb1d8f922682", "date": "2023-09-04 10:39:19 -0400", "subject": "Improve GCS multi-part performance.", "body": "The prior code avoided uploading a chunk if it was not clear whether the write was complete or not. This was primarily due to the GCS documentation being very vague on what to do in the case of a zero-size chunk.\r\n\r\nNow chunks are uploaded as they are available. This should improve performance and also reduces the diff against a future commit that absolutely requires zero-size chunks." }, { "commit": "fd9c6b0e9ddd38de57475d1325bc53b79833a983", "date": "2023-09-01 12:03:39 -0400", "subject": "Add missing const qualifier to errorTypeList." }, { "commit": "4f52015b14a1f4e5df9373707179a5447cb5a4a6", "date": "2023-08-31 20:22:01 -0400", "subject": "New CI container build for PostgreSQL 16 rc1." }, { "commit": "e3fcd8cd9e0be42a32fc4cbd2ae3db2c072eabcd", "date": "2023-08-31 16:36:52 -0400", "subject": "Add backup size completed/total to info command JSON output.", "body": "This allows for backup progress to be monitored by external processes, whereas before backup progress was only available via text output." }, { "commit": "fbd992adc7b18db5e70774d675c160c8c8e2f031", "date": "2023-08-29 13:49:49 -0400", "subject": "Allow archive-get command to run when stanza is stopped.", "body": "The restore command can run while the stanza is stopped so it makes sense for the archive-get command to follow the same rule.\n\nThe important thing is to ensure that all commands that write to the repository are stopped when the stanza is stopped." }, { "commit": "8424737697edfee7cbea195b3bccb4077e1c2ee5", "date": "2023-08-29 12:35:29 -0400", "subject": "Encode key in HTTP query.", "body": "The key also needs to be encoded (not just the value).\n\nThis is not currently an issue because none of the keys in use require encoding." }, { "commit": "d24180e4da49a8e2ef2a10a02e76cd204fe1ae8f", "date": "2023-08-29 12:28:02 -0400", "subject": "Fix conflicting group in test containers.", "body": "The tests expect the group name/id to match between the host system and the container. If there is a conflict rename the group with the required id to the expected name.\n\nThis could have unintended consequences but it seems reasonably safe since we control everything that runs in the container and there should never be any system processes running." }, { "commit": "1e6efd73e1fcf7bbe3d3f04d925fefde44847109", "date": "2023-08-23 12:56:49 -0400", "subject": "Refactor IoRead handling in backupFile().", "body": "Store the value of storageReadIo() rather than calling it each time. This is slightly more efficient, but more importantly it will be needed for an upcoming commit." }, { "commit": "82c12fed123df057fce109757aa5c72dce605440", "date": "2023-08-21 13:10:48 -0400", "subject": "Adjust range check per Coverity complaint.", "body": "Coverity complained that: Overrunning array \"optionGroupIndexKeep[optionGroupIdx]\" of 512 bytes at byte offset 4294967550 using index \"keyIdx\" (which evaluates to 4294967294).\n\nThis does not seem possible but adjust the code to make Coverity happy, as usual." }, { "commit": "eb32d6de5e1575a37efb166abbf7df400d93352e", "date": "2023-08-08 08:41:34 +0100", "subject": "Accept leading tilde in paths for SFTP public/private keys.", "body": "The documentation indicates that leading tilde file paths for public/private keys are valid but the functionality was omitted from the original implementation." }, { "commit": "1141dc20708cf0c68296f83e74c872a7db9c2bd9", "date": "2023-08-07 17:03:09 +0100", "subject": "Multi-stanza check command.", "body": "Check command now checks multiple stanzas when the stanza option is omitted.\r\n\r\nThe stanza list is extracted from the current configuration rather than scanning the repository like the info command. Scanning the repository is a problem because configuration for each stanza may not be present in the current configuration. Since this functionality is new for check there is no regression.\r\n\r\nAdd a new section to the user guide to cover multi-stanza configuration and provide additional coverage for this feature.\r\n\r\nAlso fix a small issue in the parser when an indexed option has a dependency on a non-indexed option. There were no examples of this case in the previous configuration." }, { "commit": "995a8e96691d23eaf3914a425669f89deb3ce002", "date": "2023-08-07 13:30:50 +0100", "subject": "Reload GCS credentials before renewing authentication token.", "body": "This allows the service key to be updated while a command is running. The new key should be written atomically and ideally the old key should remain valid for some period of time to avoid a race condition if the old token happens to expire at the same time that the new key is being generated." }, { "commit": "cbafcfabf2eb593de9ee95483da93f278c9ff1ee", "date": "2023-08-07 12:38:33 +0100", "subject": "Remove UNCONSTIFY() from gz compression module.", "body": "The ZLIB_CONST macro may be used since 1.2.5.2 (17 Dec 2011) to specify that the input buffers are const. This is sufficiently old to cover all non-EOL distributions, i.e. everything we test on.\n\nCompiling on older distributions may generate warnings but will continue to work." }, { "commit": "6c75c99d0b3024311ea7b254953f1c6a901bcb70", "date": "2023-07-31 11:09:17 +0200", "subject": "Update xxHash to v0.8.2.", "body": "This release fixes compilation errors on some platforms and contains numerous performance improvements." }, { "commit": "f55c8f1b813fadbc7b19a1cc4eb8f90e36b2b18c", "date": "2023-07-26 10:45:12 +0200", "subject": "Rename HRN_PG_CONTROL_OVERRIDE_PUT().", "body": "Rename to HRN_PG_CONTROL_OVERRIDE_VERSION_PUT() since other types of overrides are possible (and coming)." }, { "commit": "65fb3b3b2d89e2fc80698e52a8378395b18e6d2a", "date": "2023-07-24 09:17:46 +0200", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "c585ddaf48b806b5d038f0c90e7848875f7a533e", "date": "2023-07-24 09:15:38 +0200", "subject": "Begin v2.48 development." }, { "commit": "b5b033cc1ce152d387f20681f023b5e1f7c3ed3b", "date": "2023-07-24 09:12:30 +0200", "subject": "v2.47: Performance Improvements and Bug Fixes" }, { "commit": "5ed6f8df1487174985c42d6e13c367d70c2d8b8a", "date": "2023-07-18 07:35:12 +0200", "subject": "Fix spurious automatic delta backup on backup from standby.", "body": "When performing backup from standby the file sizes on the standby may not be equal to file sizes on the primary. This is because replication continues during the backup and by the time the file is copied from the standby it may have changed. Since we cap the size of all files copied from the standby this practically applies to truncation and in particular truncation of free space maps (at least, every case we have seen so far is an fsm). Free space maps are especially vulnerable since they are only partially replicated, which amplifies the difference between the primary and standby.\r\n\r\nOn an incremental backup it may look like the size has changed on the primary (because of the final size recorded by the standby in the prior backup) but the timestamp may not have changed on the primary and this will trigger a checksum delta for safety. While this has no impact on backup integrity, checksum delta incrementals can run much longer than regular incrementals and backup schedules may be impacted.\r\n\r\nThe solution is to preserve the original size in the manifest and use it to do the time/size check. In the case of backup from standby the original size will always be the size on the primary, which makes comparisons against subsequent file sizes on the primary consistent. Original size is only stored in the manifest when it differs from final size, so there should not be any noticeable manifest bloat." }, { "commit": "4c27d74bbd3a44d8ab1af140277d3cd8302060c2", "date": "2023-07-17 22:37:02 +0200", "subject": "Preserve block incremental info in manifest during delta backup.", "body": "It was possible for block incremental info to be lost if a file had been modified in such a way that block incremental would be disabled if the file were new, e.g. if the file shrank below the block incremental limit or the file timestamp regressed far enough into the past. In those cases the block incremental info would not be copied in manifestBuildIncr().\r\n\r\nInstead always copy the block incremental info in case the file ends up being referenced to a prior backup.\r\n\r\nThe validation tests were not robust enough to catch this issue so they were improved in 1d42aed.\r\n\r\nIn the particular case that exposed this bug, a file had a timestamp that was almost four weeks in the past at full backup time. A few days later a fail over occurred and the next incremental ran on the new primary (old standby) in delta mode. The same file had a timestamp just a few hours older than in the full backup, but now four weeks older than the current backup. Block incremental was disabled for the file on initial manifest build because of its age, which meant the block incremental info was not copied into the new manifest. The delta then determined the file had not changed and referenced it to the full backup. On restore, the file appeared to be a normal file stored in a bundle but could not be decompressed because it was in fact a block incremental." }, { "commit": "cdb7e26350822b55c00771d6d575242b1202e741", "date": "2023-07-14 17:53:58 +0300", "subject": "Fix block incremental file names in verify command.", "body": "The verify command was not appending the .pgbi extension instead of the compression extension when verifying block incremental files stored outside a bundle.\r\n\r\nOriginally the idea was that verify would not need any changes (since it just examines repo-size and checksum) but at some point the new extension was added and broke that assumption.\r\n\r\nUse backupFileRepoPathP() to generate the correct filename (Just like backup, restore, etc)." }, { "commit": "1d42aed152563fff6ec70c517d86eb39158b3f20", "date": "2023-07-14 11:18:25 +0300", "subject": "Improve validation of referenced files in command/backup unit test.", "body": "Referenced files were not being checked for validity unless they were hard linked to the current backup (which a lot of the tests did). Newer tests with bundling do not have hard links and so missed these checks.\n\nImprove the validation code to check referenced files in the manifest even when they are not hard linked into the current backup.\n\nAdd a delta test for bundling/block incremental that includes a file old enough to get a block size of zero. These are good tests by themselves but they also reduce the churn in an upcoming bug fix." }, { "commit": "c9703b35304c66a0bd0ebaaaebb389150ef207d6", "date": "2023-07-13 13:47:27 +0300", "subject": "Log when file size changes during backup.", "body": "It is possible for the size of a file to change during the backup. In most cases we won't notice since files sizes are usually capped but it is possible for some files to grow or shrink between when they are recorded in the manifest and when they are actually copied. The new size is recorded in the manifest but the old size may be useful for debugging.\n\nThe new code has coverage but no test changes because it is covered by the parallel backup testing, which does not have deterministic log output. It doesn't seem worth creating a new test to check the log format as it is not very critical (though the output was manually verified)." }, { "commit": "b6b13bd6344a1f3ed7279f5a8da771643c0e948a", "date": "2023-07-12 13:23:16 +0300", "subject": "Mention block-level backups in feature list." }, { "commit": "c8afbed5308d31302c5853850e4ed56d48b6bb4b", "date": "2023-07-12 13:09:34 +0300", "subject": "Increase protocolServerError() logging to debug level.", "body": "Errors should be rare enough that it makes sense to log them at debug level. Right now if there is an error if won't be logged at debug level, which makes it harder to tell why the main process may have terminated the local/remote process." }, { "commit": "06536b5814f2fed3e710d1d7b578aa6be53195cb", "date": "2023-07-12 10:20:09 +0300", "subject": "Rename macros in command/verify unit test.", "body": "These macros were redefined, which worked since they were identical to their prior definitions, but this will not always be true." }, { "commit": "aa229a1deef3afeebb66db1443b19e24559adf96", "date": "2023-07-09 21:56:05 +0300", "subject": "Modify time_t casts based on complaints from Coverity.", "body": "Coverity complained about time_t being cast directly to unsigned int, so instead cast the result of the operation.\n\nWe are confident in both cases that the time_t values will not be out of unsigned int range but Coverity has no way to know that.\n\nOne of these is new (introduced by 9efd5cd0) but the other one (from a9867cb0) remained unnoticed for a while, though it has not caused any production impact." }, { "commit": "28b6b2d4659d78ce67c1bd3104d96e5705d955c9", "date": "2023-07-07 10:36:15 +0200", "subject": "Improve performance of SFTP storage driver.", "body": "The initial implementation used simple waits when having to loop due to getting a LIBSSH2_ERROR_EAGAIN, but we don't want to just wait some amount of time, we want to wait until we're able to read or write on the fd that we would have blocked on.\r\n\r\nThis change removes all of the wait code from the SFTP driver and changes the loops to call the newly introduced storageSftpWaitFd(), which in turn checks with libssh2 to determine the appropriate direction to wait on (read, write, or both) and then calls fdReady() to perform the wait using the provided timeout.\r\n\r\nThis also removes the need to pass ioSession or timeout down into the SFTP read/write code." }, { "commit": "125676ae0e98d3730d2dc0d47e37ca1df145d044", "date": "2023-07-07 09:56:26 +0200", "subject": "Cleanup of init/handshake in storageSftpNew().", "body": "Rename handshakeStatus to rc to be consistent with the rest of the module.\n\nAdd comments and do some reformatting." }, { "commit": "9efd5cd0bb40c5dc2fcac2e21c9b62e554b94db6", "date": "2023-07-06 18:46:31 +0200", "subject": "Add timezone offset to info command date/time output.", "body": "This makes it easier to use timestamps from the info command directly in PostgreSQL recovery settings, especially the --target option." }, { "commit": "762498f4cd5f1425db4c176fc707926d62cc7c04", "date": "2023-07-06 11:27:00 +0200", "subject": "Fix time-based archive expiration when no backups are expired.", "body": "In the case that no backups were expired but time-based retention was met no archive expiration would occur and the following would be logged:\r\n\r\nINFO: time-based archive retention not met - archive logs will not be expired\r\n\r\nIn most cases this was harmless, but when retention was first met or if retention was increased, it would require one additional backup to expire earlier WAL. After that expiration worked as normal.\r\n\r\nEven once expiration was working normally the message would continue to be output, which was pretty misleading since retention had been met, even though there was nothing to do.\r\n\r\nBring this code in line with count-based retention, i.e. always log what should be expired at detail level (even if nothing will be expired) and then log info about what was expired (even if nothing is expired). For example:\r\n\r\nDETAIL: repo1: 11-1 archive retention on backup 20181119-152138F, start = 000000010000000000000002\r\nINFO: repo1: 11-1 no archive to remove" }, { "commit": "e280ed9098a6f5da9bdabb541e9c69d4f24ffcf2", "date": "2023-07-06 08:58:16 +0200", "subject": "Make result code handling in storage/sftp more consistent.", "body": "Initializers are useless since rc is always set later. Make rc checks consistent with the rest of the module." }, { "commit": "1fd8845c7ffd76ee335ec7960ccd531eef43bd29", "date": "2023-06-29 11:08:58 +0200", "subject": "Remove unresolved todo from 87087fac.", "body": "Seems easiest just to make the additional config required since it tests that custom ports are being used correctly. The test for synthetic was a noop since SFTP is not used in synthetic tests." }, { "commit": "0051d7ca872c84231dd55057586f957aa2bc1fa3", "date": "2023-06-29 09:42:12 +0200", "subject": "Update comments for removal of chunking and block numbers.", "body": "dd4e526 should have updated this comment but failed to do so." }, { "commit": "0ac09344dc17fb54e250705ce49399b86329bb3d", "date": "2023-06-28 18:19:20 +0200", "subject": "Fix unique label generation for diff/incr backup.", "body": "If there were at least two full backups and the last one was expired, it was impossible to make either a differential or incremental backup without first making a new full backup. The backupLabelCreate() function identified this situation as clock skew because the new backup label was compared with label of the expired full backup.\r\n\r\nIf the new backup is differential or incremental, then its label is now compared with the labels of differential or incremental backups related to the same full backup.\r\n\r\nAlso convert a hard-coded date length to a macro." }, { "commit": "5cbef3ade2ab08d8d2431f40d09683efe55c9586", "date": "2023-06-25 17:36:57 +0200", "subject": "Fix missed memory auditing in FUNCTION_LOG_RETURN_VOID().", "body": "9ca492c missed adding auditing to this macro and as a result a few memory leaks have slipped through. Add auditing to the macro to close this hole.\n\nOf the leaks found the only possibly serious one is in blockIncrProcess(), which would leak a PackRead of about eight bytes with every superblock. Since superblocks max out at a few thousand per file this was probably not too bad.\n\nAlso change the ordering of auditing in FUNCTION_TEST_RETURN_VOID(). Even though the order does not matter, having it different from the other macros is confusing and looks like an error." }, { "commit": "ecae001653489ac734c3424ea10393989c218ab5", "date": "2023-06-24 10:18:29 +0200", "subject": "Add test for configuration hash type override behavior.", "body": "This behavior is different than regular options where a repeated value will result in an error. It appears to be a legacy of the original Perl implementation, which used a hash as the underlying data type in the built-in command-line parser, and the C command-line parser was written to match." }, { "commit": "434938e32b2f71301196f933c3b9a62ed1183c31", "date": "2023-06-22 18:23:06 +0200", "subject": "Update Fedora test image to Fedora 38.", "body": "This adds testing for the latest tool versions, e.g. gcc." }, { "commit": "5531e2662d2abcd52c24f25b4a5836c9e07a8ac9", "date": "2023-06-18 12:55:29 +0300", "subject": "Add --tz param missed in C unit test migration.", "body": "This was missed in the C unit test migration and since then a new test was added that was not setting its timezone correctly.\n\nThis feature exists to make sure the tests will run on systems with different timezones and has no impact on the core code." }, { "commit": "a36f480c85c7add6fdf0e5517748d740e3a7b424", "date": "2023-06-14 14:39:40 +0200", "subject": "Clarify preference to install from packages in the user guide." }, { "commit": "9b1bc7d11bdea9cdb1bbf1210d66d53c15d72a46", "date": "2023-06-14 12:28:00 +0200", "subject": "Use new LibXML error handler.", "body": "The old error handler has been deprecated." }, { "commit": "4adf6eed09da3f0819abef813c5a44deb9c91487", "date": "2023-06-12 23:10:14 +0200", "subject": "Clarify comments for the handling of pg_control during restore." }, { "commit": "396e237d53a94b04878bfea23b4de2400256de82", "date": "2023-06-12 15:41:39 +0200", "subject": "Update NEWS.md for new version and features." }, { "commit": "6875358d11828cbbaffb51f2297a866af32ee8a4", "date": "2023-06-12 11:26:03 +0200", "subject": "Fix typo in \"Upgrading PostgreSQL\" section of user guide." }, { "commit": "818ef4b71d3ec77c0003ea8ca83981977b4447ac", "date": "2023-06-08 16:36:25 +0300", "subject": "Improve comments in protocol/command module." }, { "commit": "0bf1b4a3b8cc6caf12a4174cd52a1782cda4dc31", "date": "2023-05-26 19:50:08 +0300", "subject": "Move delete stanza section in user guide.", "body": "Deleting a stanza after all the storage driver stanzas were created was causing problems because the SFTP driver is slow and the GCS driver has no server (so it threw errors). This delayed the shutdown of PostgreSQL, which for some reason caused systemctl to hang when the documentation was being built on a RHEL host.\n\nMove the section up and add a comment about why the location is required. Also add a comment to the GCS section about its location.\n\nThis does not address the issue of systemctl hanging on RHEL container hosts but it will hopefully make it less common." }, { "commit": "5d671c63d898e9bf1c08c8ed9e830070361c861e", "date": "2023-05-25 20:08:12 +0300", "subject": "New CI container build for PostgreSQL 16 beta1." }, { "commit": "9cceed6ac4a32c14a797eba89a4ad1dc06d89d80", "date": "2023-05-24 16:34:21 +0300", "subject": "Skip recovery.signal for PostgreSQL >= 12 when recovery type=none.", "body": "Bring PostgreSQL >= 12 behavior in line with other versions when recovery type=none.\r\n\r\nWe are fairly sure this did not work correctly when PostgreSQL 12 was released, but apparently the issue has been fixed since then. Either way, after testing we have determined that the behavior is now as expected." }, { "commit": "36ff81dc6ff7e98e40c0c05a4a343a71b91b1678", "date": "2023-05-24 14:17:07 +0300", "subject": "Centralize error handling for unsupported features.", "body": "Some features are conditionally compiled into pgBackRest (e.g. lz4). Previously checking to see if the feature existed was the responsibility of the feature's module.\r\n\r\nCentralize this logic in the config/parse module to make the errors more detailed and consistent.\r\n\r\nThis also fixes the assert that is thrown when SFTP storage is specified but SFTP support is not compiled into pgBackRest." }, { "commit": "de46276bf60b8a2181621b123d1a99e56808a089", "date": "2023-05-24 09:56:32 +0300", "subject": "Refactor allow list processing in config/parse module.", "body": "Combine StringId and int checking into a single loop. This seems more compact and makes it easier to add code that affects both types (and possibly more types in the future)." }, { "commit": "48b26bf569664da59bff08eb3ef959f0cd440e29", "date": "2023-05-23 12:50:12 +0300", "subject": "Fix sftp-all=y in the user guide so it creates a valid configuration.", "body": "This was not tested in 87087fac and the generated config was only valid for pushing from the primary. Also do some general cleanup.\n\nUpdate the SFTP server user to be \"pgbackrest\" instead of \"postgres\".\n\nEven though sftp-all=y now creates a valid configuration, the user guide build still fails because SFTP is too slow and operations time out (particularly starting PostgreSQL). This will need to be addressed in a future commit." }, { "commit": "c633b187db32d18ccab24e77ab3413335e067775", "date": "2023-05-23 10:58:51 +0300", "subject": "Remove user-facing documentation references to --vm=none.", "body": "This parameter is now optional and defaults to none so there is no reason to explicitly show it in user-facing documentation.\n\nAlso make the vm parameter in ci.pl optional to be consistent with how test.pl behaves." }, { "commit": "4ec51cdb2f92e3fd814063a2bb606640965f8c5a", "date": "2023-05-23 10:40:56 +0300", "subject": "Explicitly create SSH directory in SFTP setup.", "body": "RHEL 9 (at least) will warn on stderr that the directory has been created which causes the documentation build to fail." }, { "commit": "5bbe98758919a38692cd859809203cb029543c59", "date": "2023-05-23 10:14:00 +0300", "subject": "Build u20 image to speed contributing document generation.", "body": "This image was left out of the last round of builds, which forced the contributing document to build it from scratch." }, { "commit": "c2c60350d376379322d6c14c27cbb93bf6e86987", "date": "2023-05-23 08:25:17 +0300", "subject": "Add missed --no-log-timestamp in unit tests and improved formatting.", "body": "The --no-log-timestamp option was missed when unit test building was migrated to C, which caused test timings to show up in the contributing guide. This caused no harm but did create churn in this file during releases.\n\nAlso improve the formatting when test timing is disabled." }, { "commit": "9dbf76d8e87287786f03b65c94305b3dc126029b", "date": "2023-05-22 11:19:00 +0300", "subject": "Begin v2.47 development." }, { "commit": "b461f7c6f884dba8056858464d1b571e708dad77", "date": "2023-05-22 11:13:13 +0300", "subject": "v2.46: Block Incremental Backup and SFTP Storage" }, { "commit": "a28b72c62431bcf9df02791219d9d6f5d0e60b54", "date": "2023-05-18 09:32:44 +0300", "subject": "Add warning about using recovery type=none." }, { "commit": "c61115e8adccd17d8afb6bd527fb6a396ec6b72a", "date": "2023-05-15 17:46:41 +0300", "subject": "Add bundling and block incremental to stress testing.", "body": "This makes the stress testing more robust and provides additional testing for important features." }, { "commit": "87087fac231d2110160b3219daee3f9fb91aec02", "date": "2023-05-13 19:16:16 +0300", "subject": "SFTP support for repository storage." }, { "commit": "0c1f823e7a3abf1cadb44de3832ec1ddb2da7ec8", "date": "2023-05-12 15:46:04 +0300", "subject": "Add note about running stanza-create on already-created repositories." }, { "commit": "c7ac17330405510e56243e967d108b8d3b387ce0", "date": "2023-05-10 09:22:26 +0300", "subject": "Increase check timeouts in real/all integration tests.", "body": "The prior timeouts were a bit aggressive and were causing timeouts in the Azure tests. There have also been occasional timeouts in other storage drivers.\n\nThe performance of CI environments is pretty variable so increased timeouts should make the tests more stable." }, { "commit": "2fc641383709ce1f91f35bb1edf4e7c1ab0f4eb3", "date": "2023-05-06 12:16:58 +0300", "subject": "Fix typos in test titles." }, { "commit": "6f250bf6860054397181f5678220faaa6574993e", "date": "2023-05-06 12:15:21 +0300", "subject": "Fix groupId type in StorageInfo.", "body": "gid_t is the correct type but there was no bug because uid_t and gid_t have the same underlying type." }, { "commit": "ad9196cb49ce86cef678e453046803fb2be2cf70", "date": "2023-05-05 10:08:49 +0300", "subject": "Update Debian user guide to build on Ubuntu 20.04.", "body": "Ubuntu 18.04 will be EOL at the end of the month so update to a newer version." }, { "commit": "023d3c99d8d00497501b21fb20b07f71ea49ee40", "date": "2023-05-04 08:52:31 +0300", "subject": "Fix incorrect commit hash in 2f322e2a." }, { "commit": "2f322e2a49f20ff3036db6382363efb40246922e", "date": "2023-05-03 09:49:24 +0300", "subject": "Add efac3474 (remove double spacing) to .git-blame-ignore-revs." }, { "commit": "d49907239eb37c3e4e905f97543824181a1bd406", "date": "2023-05-03 09:37:59 +0300", "subject": "Remove double spaces missed in 1bd5530a." }, { "commit": "efac3474cffc4f01154c13cc358601c4c98bcd42", "date": "2023-05-02 12:59:20 +0300", "subject": "Add 1bd5530a (remove double spacing) to .git-blame-ignore-revs." }, { "commit": "1bd5530a59cd8ddbabc279802d1ede4f8fbd5314", "date": "2023-05-02 12:57:12 +0300", "subject": "Remove double spaces from comments and documentation.", "body": "Double spaces have fallen out of favor in recent years because they no longer contribute to readability.\n\nWe have been using single spaces and editing related paragraphs for some time, but now it seems best to update the remaining instances to avoid churn in unrelated commits and to make it clearer what spacing contributors should use." }, { "commit": "5ea7b91bf9ab57a29778d2332c0971e84f14c4c6", "date": "2023-05-02 12:07:28 +0300", "subject": "Remove extraneous space from error messages." }, { "commit": "9637d9449091632f158f8e2be8afbad4bda200fc", "date": "2023-05-02 09:37:42 +0300", "subject": "Use boolean variant constants where possible instead of redeclaring." }, { "commit": "822f2a584260a012b833455a7a2bd1c8487e3010", "date": "2023-05-01 19:26:48 +0300", "subject": "Finalize block incremental feature.", "body": "Remove beta status and update documentation to remove beta references and warnings.\r\n\r\nThe repo-block-* sub-options have been marked internal. Most users will be best off with the default behavior and we may still decide to change these options for remove them in the future." }, { "commit": "c510046dd77c40487d2d6c659834927a3bab1106", "date": "2023-05-01 14:27:56 +0300", "subject": "Remove extraneous spaces in user guide." }, { "commit": "dd4e52679ecdc43539bd038b5aa9bd4ff9055cbf", "date": "2023-04-27 23:29:12 +0300", "subject": "Removing chunking and block numbers from incremental block list.", "body": "These were intended to allow the block list to be scanned without reading the map but were never utilized. They were left in \"just in case\" and because they did not seem to be doing any harm.\r\n\r\nIn fact, it is better not to have the block numbers because this allows us set the block size at a future time as long as it is a factor of the super block size. One way this could be useful is to store older files without super blocks or a map in the full backup and then build a map for them if the file gets modified in a diff/incr backup. This would require reading the file from the full backup to build the map but it would be more space efficient and we could make more intelligent decisions about block size. It would also be possible to change the block size even if one had already been selected in a prior backup.\r\n\r\nOmitting the block numbers makes the chunking unnecessary since there is now no way to make sense of the block list without the map. Also, we might want to build maps for unchunked block lists, i.e. files that were copied normally." }, { "commit": "3fc3690dd77ef5fa032abb006558184164db53ec", "date": "2023-04-27 10:30:50 +0300", "subject": "PostgreSQL 16 support.", "body": "Add catalog version and WAL magic for PostgreSQL 16.\r\n\r\nThe GUC to force parallel mode has be renamed so update that in the tests." }, { "commit": "39059dc4e7a74e6473099eefeed4e9e0efe65e60", "date": "2023-04-26 15:00:36 +0300", "subject": "Remove extra linefeed." }, { "commit": "3ff88ffbb495fd669e9a361478e60192930ee0a1", "date": "2023-04-25 11:52:28 +0300", "subject": "Avoid chown() on recovery files during restore.", "body": "The chown() was already skipped on the files restored from the repository but the same logic was not applied to the generated recovery files, probably because chown'ing a few recovery files does not have performance implications. Use the same logic for recovery files to determined if they need to be chown'd.\r\n\r\nUltimately this behavior is pretty hard to test, so add a fail safe into the Posix driver that will skip chown if the permissions are already as required." }, { "commit": "750ab8e55c276fa0127d19961a59fec254e24c61", "date": "2023-04-21 16:21:10 +0300", "subject": "Add MacOS back to CirrusCI.", "body": "9e29c01 removed MacOS testing due to breaking changes in the update to arm on the platform.\n\nUpdate the scripts to correctly work with the version of Homebrew deployed with the arm images." }, { "commit": "23d7d67d4961c9bb38d73ef1c409369303276846", "date": "2023-04-21 15:08:59 +0300", "subject": "Fix missing void parameter." }, { "commit": "f5e6bc26983396ec51c6579aade37db39807f3a5", "date": "2023-04-20 13:24:12 +0300", "subject": "Allow page header checks to be skipped.", "body": "These checks cause false negatives for page checksum verification when the page is encrypted because pd_upper might end up as 0 in the encrypted data. This issue is rare but reproducible given a large enough cluster.\r\n\r\nMake these checks optional, but leave them enabled by default." }, { "commit": "8240eb5da57d2950ebbd3492c163b4e0151d3cb7", "date": "2023-04-16 17:41:27 +0300", "subject": "Autogenerate PostgreSQL versions.", "body": "This will make adding/removing versions of PostgreSQL more reliable." }, { "commit": "a05bf6bb15ba7f7e4b610dbb84e87ebcd359dd38", "date": "2023-04-16 17:32:24 +0300", "subject": "Rename PG_VERSION_*_STR constants to PG_VERSION_*_Z.", "body": "This is more consistent with other zero-terminated string constants and also has the benefit of being shorter." }, { "commit": "75254c9285e70913920af9ab6e9f362bc4d11d7d", "date": "2023-04-11 16:28:29 +0400", "subject": "Parameterize configLoad() as cfgLoadP().", "body": "There is one existing optional parameter and there are more to come." }, { "commit": "d6cb3de17afff0a9c57f312455a5ca3cf202cba6", "date": "2023-04-11 14:50:59 +0400", "subject": "Update command/check module to recent coding standards.", "body": "Add const as appropriate and remove an unneeded declaration." }, { "commit": "a9c5cd9749f6b5ae1005453b1039d91f37fb245b", "date": "2023-04-11 12:46:00 +0400", "subject": "Update command/backup module to recent coding standards.", "body": "Add const as appropriate and avoid setting results to NULL if the result will definitely be set later on." }, { "commit": "f33e1cfb1600643820a1ae76bdeb15c62ac1fcc7", "date": "2023-04-06 11:38:18 +0400", "subject": "Add error retry detail for HTTP retries.", "body": "This should make it clearer when retries have happened and for how long." }, { "commit": "df419b34b55a15a902ff5cc1912aaebc7e023738", "date": "2023-04-06 11:03:46 +0400", "subject": "Improve retry error messages.", "body": "Centralize the code to allow it to be used in more places and update the protocol/server module to use the new code.\n\nSince the time measurements make testing difficult, also add time and errorRetry harnesses to allow specific data to be used for testing. In the case of errorRetry, the production behavior is turned off by default during testing and only enabled for the errorRetry test module." }, { "commit": "801e396dace2a4d9f4125ef0e751c90dc0cc644a", "date": "2023-04-06 10:38:49 +0400", "subject": "Move error modules to common/error directory.", "body": "There are already enough files to warrant a separate directory and more are coming.\n\nAlso remove extraneous includes." }, { "commit": "ff98636e41b8ad14babe1e9fa1fdc15aad2244cd", "date": "2023-04-06 08:22:14 +0400", "subject": "Update 32-bit CI host to Debian 10 from Ubuntu 18.04.", "body": "Ubuntu 18.04 will be EOL before the next release, so update to the oldest available Debian version.\n\nAlso fix one incorrect return value type, a test cast, and adjust some test timeouts." }, { "commit": "a9f39857cf7d8c0eda9f5f19f3d4c9497d4d47f2", "date": "2023-04-05 11:24:53 +0400", "subject": "Remove unused strOS parameter in Common::HostTest." }, { "commit": "8f7f73e4af916bf1bc6a0cb72be35b7061b297df", "date": "2023-03-30 17:46:25 +0500", "subject": "Sleep using nanosleep() instead of select().", "body": "This is a safer way to sleep due to select's not-portable interaction with signals.\n\nBased on https://git.postgresql.org/gitweb/?p=postgresql.git;a=commitdiff;h=a948e49e2ef11815be0b211723bfc5b53b7f75a8 from the PostgreSQL project." }, { "commit": "2cfbee903a0a16d1fdeca52dbf6e775cdcddcc74", "date": "2023-03-30 15:43:37 +0500", "subject": "Use Buffer object for page buffer in PageChecksum filter.", "body": "This avoids the object needing to support allocations." }, { "commit": "3dbf518c3a9df538ca110039758fbfe161e3f714", "date": "2023-03-30 14:50:46 +0500", "subject": "Convert XmlNode from allocation to object.", "body": "This makes memory accounting more accurate and objects that use Xml no longer need to allow allocations." }, { "commit": "047472144b5fd92425225700aa11b351044244e5", "date": "2023-03-30 14:42:20 +0500", "subject": "Remove allocations from common crypto module.", "body": "The allocations are better done with a struct and Buffer object." }, { "commit": "84ca7b9b2713b6d36d15d592e78942d8e1ac8a35", "date": "2023-03-30 11:42:22 +0500", "subject": "Cleanup for objects that do not require allocations.", "body": "A lot of these are left over from when object interfaces required allocations (changed in f6e30736 and 9ca9c8e4). Others are likely copy/paste errors.\n\nThis saves some space in the mem context and makes it clear that no allocations will be made." }, { "commit": "fe0fd71be2e9754491cbbdcde79e7a7cc87f2e7b", "date": "2023-03-30 10:00:58 +0500", "subject": "Convert the result of zNewFmt() into an object.", "body": "The result is not intended to be freed directly so this makes memory tracking more accurate. Fix a few places where memory was leaking after a call to zNewFmt().\n\nAlso update an assert to make it clearer." }, { "commit": "295b53e8453b4ac4889b62bc1c7f1d6b78b09421", "date": "2023-03-28 16:24:43 +0600", "subject": "Update Github actions to new versions.", "body": "These updates remove various deprecation warnings." }, { "commit": "b111599badccb1c196a224e3d6ac8657b8d172a2", "date": "2023-03-28 15:05:18 +0600", "subject": "Simplify object creation with OBJ_NEW_BEGIN() macro.", "body": "Eliminate the boilerplate of declaring this and assigning memory to it, which is the same for the vast majority of object creations.\n\nKeep the old version of the macro as OBJ_NEW_BASE_BEGIN() for a few exceptions in the core code and (mostly) in the tests." }, { "commit": "91f9301b9d0026ee72875d943e0ed01b14ec887b", "date": "2023-03-27 16:42:35 +0600", "subject": "Fix command/manifest unit test title." }, { "commit": "9ca9c8e4c97b60a4cceef1c6f9fcb93e413e15d7", "date": "2023-03-27 14:32:37 +0600", "subject": "Improve interface handling in remaining modules.", "body": "As in f6e30736, make the interface object the parent of the driver object rather than the interface being allocated directly in the driver object. Allow exceptions to this general rule for objects that need to retain ownership of their interfaces." }, { "commit": "5f001248cce08137a601ce5671aa67d26bfdd366", "date": "2023-03-25 14:59:47 +0700", "subject": "Fix indentation in config.yaml." }, { "commit": "8ff956ad7e8608dd7582825521105c289b249810", "date": "2023-03-25 14:07:31 +0700", "subject": "Add lock module initialization.", "body": "Each call to lockAcquireP() passed enough information to initialize the lock system. This was somewhat inefficient and as locks become more complicated it will lead to more code duplication. Since a process can only take one type of lock it makes sense to do most of the initialization up front.\n\nAlso reduce the log level of lockRelease() since it is only called at exit and the lock will be released in any case." }, { "commit": "f1caecc4ff38557cf81314738ac83d420d8bcf23", "date": "2023-03-24 10:34:42 +0800", "subject": "Convert lockAcquire() to lockAcquireP().", "body": "This makes a few current parameters optional and allows for more optional parameters with less code churn." }, { "commit": "c8ec114c8cb392d227fa267c62d11cc6b4fb9677", "date": "2023-03-21 12:29:45 +0800", "subject": "Add reference filter and output to manifest command.", "body": "This allows the file list to be filtered by reference. The reference is output when it is not the default reference for the backup." }, { "commit": "1807bfcff5550bb40fa4f92dee1e15580a0b752d", "date": "2023-03-21 12:22:47 +0800", "subject": "Clear buffer limit when buffer is limited to allocated size.", "body": "This minor optimization automatically clears the limit flag when limit is set to allocated size. This has no impact on the current code but will simplify a future commit where a conditional bufLimitClear() is being used." }, { "commit": "5b5786c082b698e637bd44b9f06fbca5e4e4cc21", "date": "2023-03-20 10:25:44 +0800", "subject": "Begin v2.46 development." }, { "commit": "6ad79d16ca638b7ccdc81b5530a39007b8e01aed", "date": "2023-03-20 09:37:23 +0800", "subject": "v2.45: Block Incremental Backup (BETA)" }, { "commit": "04d224b88e73194ff1c6497dbecaaff392008dfe", "date": "2023-03-18 20:47:03 +0800", "subject": "Add beta feature and documentation for block incremental backup." }, { "commit": "78c036efb7ebf7060949f3d2f2e6d93986734414", "date": "2023-03-16 17:46:32 +0700", "subject": "Updates required for new manifest command to work remotely.", "body": "Additional options are required and the BlockChecksum must work remotely." }, { "commit": "505a639f1e627b506c2e5a3351e8d4aacae9ed0b", "date": "2023-03-14 21:56:05 +0700", "subject": "Prototype manifest command.", "body": "Output a manifest in text or JSON format. Neither format is complete but they cover the basics.\n\nIn particular the manifest command outputs the complete block restore when the filter option is specified and the block delta when the pg option is also specified. This is non-destructive so it is safe to execute against a running cluster." }, { "commit": "c30d3e439b77d544477e289606d30d0c8ffe301f", "date": "2023-03-14 17:48:25 +0700", "subject": "Block incremental map fixes and improvements.", "body": "Bug Fixes:\r\n\r\n* Remove the distinction between maps where super block size is equal to block size and maps where they are not. In practice, maps with equal blocks are now rare and most of the optimizations can be applied directly to super blocks where the blocks are equal. This fixes a bug where a map that was created with equal size blocks and then converted to differing block sizes would generate an invalid map.\r\n\r\n* Free reads during restore to avoid running out of file handles.\r\n\r\nImprovements:\r\n\r\n* Store super block sizes in the block map. This allows the final block size to be removed from the block list and provides a more optimal restore and better potential for analysis.\r\n\r\n* Always round the super block size up to the next block size. This makes the number of blocks per super block more predictable.\r\n\r\n* Allow super block sizes to be changed at will in the map. The first case for this is to store the reduced super block size required when the last super block is short but it could be used to dynamically change the super block size to optimize compression.\r\n\r\n* Store a block count rather than a list of blocks in a super block. Blocks must always be sequential, though there may be an offset to the first block in a super block. This saves 11-14% on space for checksum sizes 6-7.\r\n\r\n* In the case that all the blocks for a super block are present, and there is no offset, the block size is omitted." }, { "commit": "5c1f78d4dd62c37218285d13b09a9450eb3a588e", "date": "2023-03-12 22:38:38 +0700", "subject": "Fix typo in blockIncrProcess()." }, { "commit": "1281a6eaf8552232c8c9af4d7fc15c082a7794d9", "date": "2023-03-12 16:21:43 +0700", "subject": "Ensure no continuations when block size equals super block size.", "body": "In this case each super block contains a single block so continuations are not possible." }, { "commit": "d7704a8bc837b0be5564737e491bf5f73796ca0a", "date": "2023-03-11 14:04:43 +0700", "subject": "Move backup pq test script generation to backup harness.", "body": "This allows scripted online backups to be used by other test modules." }, { "commit": "2fffd64213b7d51b6444f61502f4225584667731", "date": "2023-03-10 16:11:30 +0700", "subject": "Fix error handling in build-code binary.", "body": "120a49b6 improved the error handling but due to a copy/pasto errors were being handled as asserts, which tripped an assertion in the log module." }, { "commit": "19f3a1d3040df56e73f19674618eb5f447dabe8d", "date": "2023-03-10 16:08:30 +0700", "subject": "Add missing header." }, { "commit": "24f725212d0c1dedc0f34fe4e939e481585ff697", "date": "2023-03-10 15:30:27 +0700", "subject": "Add beta feature infrastructure.", "body": "This allows options to be marked as beta, which will require that the --beta option be supplied to prevent accidental usage of a beta feature.\n\nThe online and command-line documentation also show warnings when options are beta." }, { "commit": "6b409d049eb935146f7996b75b97744b75c411a9", "date": "2023-03-10 14:01:38 +0700", "subject": "Update default block size and super block values based on testing.", "body": "Block sizes are incremented when the size of the map becomes as large as a single block. This is arbitrary but it appears to give a good balance of block size vs map size.\n\nThe full backup super block size is set to minimize loss of compression efficiency since most blocks in the database will likely never be modified. For diff/incr backup super blocks, a smaller size is allowable since only modified blocks are stored. The overall savings of not storing unmodified blocks offsets the small loss in compression efficiency due to the smaller super block and allows more granular fetches during restore." }, { "commit": "1119a5353911d6d326edac325f530bb796a806ae", "date": "2023-03-09 11:04:03 +0700", "subject": "Rename BlockHash to BlockChecksum.", "body": "Checksum is the generally used terminology in the code base, even when a hash is being used as a checksum." }, { "commit": "6252c0e4485caee362edec13302a5f735a69bff4", "date": "2023-03-09 10:30:57 +0700", "subject": "Exclude backup set size from info for block incremental backups.", "body": "As calculated this size is not correct since it does not include the parts of prior block incrementals that are required to make the current block incremental valid. At best this could be approximated and the resulting values might be very confusing.\n\nFor now, at least, exclude this metric for block incremental backups." }, { "commit": "210bed4511ae5523b170eb193b877346d787140d", "date": "2023-03-09 10:02:04 +0700", "subject": "Use xxHash instead of SHA-1 for block incremental checksums.", "body": "xxHash is significantly faster than SHA-1 so this helps reduce the overhead of the feature.\n\nA variable number of bytes are used from the xxHash depending on the block size with a minimum of six bytes for the smallest block size. This keeps the maps smaller while still providing enough bits to detect block changes." }, { "commit": "8b5153ad210011812c8fe6db2f2ed829a5aa543a", "date": "2023-03-09 09:39:54 +0700", "subject": "Block-level incremental backup super blocks.", "body": "Small blocks sizes can lead to reduced compression efficiency, so allow multiple blocks to be compressed together in a super block. The disadvantage is that the super block must be read sequentially to retrieve blocks. However, different super block sizes can be used for different backup types, so the full backup super block sizes are large for compression efficiency and diff/incr are smaller for retrieval efficiency." }, { "commit": "740c2258e36613b1b4e76bab452d0e610a399e7f", "date": "2023-03-09 08:23:15 +0700", "subject": "Add pg-version-force option for fork integration.", "body": "Forks may update pg_control version or WAL magic without affecting the structures that pgBackRest depends on.\r\n\r\nThis option forces pgBackRest to treat a cluster as the specified version when it cannot be automatically identified." }, { "commit": "2fa7e53c5d4b5a5fd79f62d39763746727a5b9be", "date": "2023-03-08 19:05:23 +0700", "subject": "Skip writing recovery.signal by default for restores of offline backups.", "body": "When restoring an offline backup on PostgreSQL >= 12, skip writing recovery.signal by default since this will error if the backup was made with wal_level=minimal. If the user explicitly sets the type option to something other than none, then write recovery.signal as usual since it is possible to do Point-In-Time-Recovery from an offline backup as long as wal_level was not minimal." }, { "commit": "7e5adc03594a3b7a975001e80902a51df3ddd577", "date": "2023-03-07 18:46:24 +0700", "subject": "Use raw compression/encryption to bundling and block incremental backup.", "body": "Raw encryption was already being used for block incremental. This commit adds raw compression to block incremental where possible (see da918587).\n\nRaw compression/encryption is also added to bundling for a backup set when block incremental is enabled on the full backup. This prevents a break in backward compatibility since block incremental is not backward compatible." }, { "commit": "da9185870222057b97365f5db0acf797f959ec5e", "date": "2023-03-07 18:31:17 +0700", "subject": "Add optional raw format for compression types.", "body": "Raw format saves 12 bytes of header for gzip and 4 bytes of checksum for lz4 (plus CPU overhead). This may not seem like much, but over millions of small files or incremental blocks can really add up. Even though it may be a relatively small percentage of the overall backup size it is still objectively a large amount of data.\n\nUse raw format for protocol compression to exercise the feature.\n\nRaw compression format will be added to bundling and block incremental in a followup commit." }, { "commit": "f6e307365f5a55a8bb4eb694e0a11fb4f32fb6e7", "date": "2023-03-07 11:27:41 +0700", "subject": "Improve interface handling in storage module.", "body": "Make the interface object the parent of the driver object rather than the interface being allocated directly in the driver object.\n\nThe prior method was more efficient when mem contexts had a much higher cost. Now mem contexts are cheap so it makes more sense to structure the objects in a way that works better with mem context auditing. This also means the mem context does not need to be stored separately since it can be extracted directly from the interface object.\n\nThere are other areas that need to get the same improvement before the specialized objMoveContext() and objFreeContext() functions can be removed." }, { "commit": "120a49b6590a549a72bfe0a6a5a22905e835b2ce", "date": "2023-03-06 21:07:08 +0700", "subject": "Improved error handling in build-code binary.", "body": "Show a full stack trace instead of just the error." }, { "commit": "0818601c055724bde3b97d4b02d3a38b0a53899a", "date": "2023-03-06 09:49:33 +0700", "subject": "Set online flag in manifest in command/restore unit tests.", "body": "This flag does not currently affect restore behavior but it will in an upcoming commit. Set the flag here to simplify the test diff in the upcoming commit." }, { "commit": "c656669ac2354dab504d94b604e305c3b68bb493", "date": "2023-03-04 12:50:02 +0700", "subject": "Allow control version and WAL magic to be overridden in test harness.", "body": "This makes it easier to write tests for invalid control version/WAL magic.\n\nAlso add HRN_PG_WAL_TO_BUFFER() to simplify generation of WAL headers." }, { "commit": "1648c133d6bc1ffd0682ae6219934d00295aa580", "date": "2023-03-04 12:45:08 +0700", "subject": "Keep only one all-default group index.", "body": "It is possible for a group index to be created for an option that is later found to not meet dependencies. In this case all values would be default leading to a phantom group, which can be quite confusing.\r\n\r\nRemove group indexes that are all default (except the final one) and make sure the key for the final all default group index is 1." }, { "commit": "439447977681ea8730935535e25a657334596942", "date": "2023-02-28 08:47:51 +0700", "subject": "Fix typo and remove extraneous linefeed." }, { "commit": "16ac5ee8d3b926464cd27e8f064579f1e7935f4c", "date": "2023-02-26 16:13:44 +0700", "subject": "Rename block incremental manifest keys.", "body": "Since the keys need to be read/written in order, these keys make the logic a bit simpler." }, { "commit": "a9867cb0b8e8d7abcac9b18b76e3e490c265f3cb", "date": "2023-02-26 14:49:34 +0700", "subject": "Add repo-block-age-map and repo-block-size-map options.", "body": "Make these options configurable. This is primarily for testing purposes so the new options will be kept internal." }, { "commit": "15d5dcdd3bb387aab28bbe7262011c38c7d6d22c", "date": "2023-02-26 14:41:32 +0700", "subject": "Add explicit instructions for upgrading between 2.x versions.", "body": "Add an explicit statement that there is nothing special to do when upgrading between 2.x versions.\r\n\r\nLeave the previous paragraph about the default location that changed between 2.00 and 2.02, as it is more a matter of transitioning from 1.x to 2.x." }, { "commit": "dffc933384e4327e2fd52868dd19668ed9d3ed91", "date": "2023-02-13 09:17:30 +0700", "subject": "Rename DeltaMap to BlockHash.", "body": "This more accurately describes what the object does." }, { "commit": "779efe0d7a54340141e01597279e9ab1d0bd0773", "date": "2023-02-09 13:01:56 +0700", "subject": "Consistently declare block incremental size as size_t.", "body": "The block is often allocated in memory so size_t makes more sense than uint64_t." }, { "commit": "d520816acf9ecb01ef805cecbc9de7e8d0a9f468", "date": "2023-02-09 08:11:05 +0700", "subject": "Remove parameter list from deltaMapNew().", "body": "Since this filter cannot be used remotely (yet) there is no reason to create a parameter list." }, { "commit": "3feed389a2199454db68e446851323498b45db20", "date": "2023-02-08 22:34:23 +0700", "subject": "Improve IoChunkedRead end-of-file handling.", "body": "Determine end-of-file earlier to improve throughput.\n\nAlso clean up some comments and formatting." }, { "commit": "089fae035bf0a9778fff0527ea3cbb113625a403", "date": "2023-02-07 14:09:50 +0700", "subject": "Add block incremental to real/all test output." }, { "commit": "31cad5e09e1910fc9d36d18306524aabca3ff36b", "date": "2023-02-01 12:57:04 +0700", "subject": "Check for stray execute permissions in test.pl --code-format.", "body": "Sometimes these can get set while updating permissions from with a VM or container.\n\nAlso fix a few permissions that were not correct." }, { "commit": "8e7e9d36a133f2ec01f4864b5b0c80344888b737", "date": "2023-01-31 21:28:28 +0700", "subject": "Fix contributors in release notes." }, { "commit": "c5907a2e7193ca8f79cf56dca23e0df118e686c3", "date": "2023-01-31 08:28:32 +0700", "subject": "Remove references to SSH made obsolete when TLS was introduced.", "body": "Also remove details about SSH compression that are not helpful." }, { "commit": "9ae6b6ef0f344b49f6153bf842ac815b0fcb9ea3", "date": "2023-01-31 08:04:09 +0700", "subject": "Update comments missed in copy/paste." }, { "commit": "ce0ea2cfab17e6770de3897133183af891df943c", "date": "2023-01-30 12:22:41 +0700", "subject": "Use uncrustify for code formatting.", "body": "The code is not completely reflowed yet so there are some cases that uncrustify will not catch. The formatting will be improved over time.\n\nSome block of code require special formatting so have been surrounded with the {uncrustify-off}/{uncrustify-on} markers. These exceptions should be kept to a minimum.\n\nAdd --code-format (to reformat code) and --code-format-check (to check formatting) to test.pl.\n\nAdd a CI test that will check code formatting. Code must be correctly formatted before it can be merge to integration.\n\nAdd documentation to the coding standards for code formatting." }, { "commit": "d4070c90641a61fa3cb1169f3bd53067193bab4e", "date": "2023-01-30 11:55:54 +0700", "subject": "Reformat code with uncrustify.", "body": "uncrustify has been configured to be as close to the current format as possible but the following changes were required:\n\n* Break long struct initializiers out of function calls.\n* Bit fields get extra spacing.\n* Strings that continue from the previous line no longer indented.\n* Ternary operators that do not fit on a single line moved to the next line first.\n* Align under parens for multi-line if statements.\n* Macros in header #if blocks are no longer indented.\n* Purposeful lack of function indentation in tests has been removed.\n\nCurrently uncrustify does not completely reflow the code so there are some edge cases that might not be caught. However, this still represents a huge improvement and the formatting can be refined going forward.\n\nSupport code for uncrustify will be in a followup commit." }, { "commit": "b2202c36d9e7c4557ac37087757df80193d516b5", "date": "2023-01-30 11:16:31 +0700", "subject": "Fix formatting errors.", "body": "Errors in our current (manually-maintained) code format discovered by uncrustify." }, { "commit": "a96837d2eda4b7497e74954b24d9323f33465c50", "date": "2023-01-30 10:55:14 +0700", "subject": "Refactor loop limit in removeExpiredArchive().", "body": "This simplifies the formatting, which was a bit tortured. Also, there is no need to check the limit on each iteration so this is a bit more efficient." }, { "commit": "f91af305845aa1e99d830a7a6addaee058b518d1", "date": "2023-01-30 10:51:48 +0700", "subject": "Add FN_PRINTF() macro.", "body": "Bring the format(printf) attribute in line with the FN_NO_RETURN and FN_INLINE_ALWAYS macros.\n\nThis is simpler to read and can be customized for different compilers." }, { "commit": "f8ea51bf8f4de61fe3541e2f2701b9dccb5ede70", "date": "2023-01-30 10:27:17 +0700", "subject": "Refactor stackTraceToZ() to avoid multiple versions of the function.", "body": "stackTraceToZ() was split this way in c8264291 to allow complete coverage. 0becb6da added a shim to improve coveage but missed simplifying the function." }, { "commit": "0d8e08f69c2ece7a060bd30dff0b375d013fe2fd", "date": "2023-01-30 09:29:07 +0700", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "240312110c5b1c07deb200b06755a886b723485b", "date": "2023-01-30 09:27:04 +0700", "subject": "Begin v2.45 development." }, { "commit": "053468bfb156a8ecbd5d0325d35141056c89ec52", "date": "2023-01-30 09:15:44 +0700", "subject": "v2.44: Remove PostgreSQL 9.0/9.1/9.2 Support" }, { "commit": "a28f3d49c23d7dbd8873d07b7d513b6b7e59345a", "date": "2023-01-28 18:55:53 +0700", "subject": "Add --no-back-trace option to test.pl.", "body": "Running valgrind and backtrace together has been causing tests to timeout in CI, mostly likely due to limited resources. This has not been a problem in normal development environments.\n\nSince it is still important to run backtraces for debugging, split the u22 test that was doing all this work to run coverage and backtrace together and valgrind-only as a separate test. As a bonus these tests run faster separately and since they run in parallel the total execution time is faster." }, { "commit": "16c625353db2569ebcc0c28e09d76a9811d2a241", "date": "2023-01-28 16:22:04 +0700", "subject": "Skip test modules in coverage report.", "body": "Coverage of the test code is useful during development but it does not seem worth adding it to the core code coverage report." }, { "commit": "d5ef4ce747f0d743b76c5fe05c4e4a02cdd9bd09", "date": "2023-01-28 11:01:29 +0700", "subject": "Fix permissions on source files." }, { "commit": "3aea997df5ace85251d0579ccbe13fa938f587ee", "date": "2023-01-25 10:59:13 +0700", "subject": "Add warning about enabling \"hierarchical namespace\" on Azure storage.", "body": "If this feature is enabled expire will fail since directories need to be deleted separately.\r\n\r\nIdeally we would add support for this feature but for now we'll just document the issue." }, { "commit": "ed818b31867c640d72ef829dd36cd0a7feff12df", "date": "2023-01-25 10:35:03 +0700", "subject": "Add dark mode to the website.", "body": "The colors could use more tweaking but at least the website will no longer blind users running dark mode." }, { "commit": "1da2666a9ef41457ac64ddf2cabccc93b3b05aeb", "date": "2023-01-21 14:03:27 +0700", "subject": "Add manifest test harness.", "body": "These macros make adding paths/files/etc to a manifest simpler and easier to read." }, { "commit": "912eec63bb971c10a545e582f628c982f4e00005", "date": "2023-01-20 16:48:57 +0700", "subject": "Block-level incremental backup.", "body": "The primary goal of the block incremental backup is to save space in the repository by only storing changed parts of a file rather than the entire file. This implementation is focused on restore performance more than saving space in the repository, though there may be substantial savings depending on the workload.\r\n\r\nThe repo-block option enables the feature (when repo-bundle is already enabled). The block size is determined based on the file size and age. Very old or very small files will not use block incremental." }, { "commit": "008a18555c6474be4327bfe0830ef13166a18ece", "date": "2023-01-20 15:44:01 +0700", "subject": "Fix comment indentation and wrapping." }, { "commit": "bfc9a5c33e032ee7f95180cafcec07ce19e11aa9", "date": "2023-01-14 18:01:48 +0700", "subject": "Fix incorrect FUNCTION_TEST_END() in hrnPgWalToBuffer()." }, { "commit": "ccee5c0fb1653f00e74de13d917a24b8b3506c22", "date": "2023-01-14 17:12:15 +0700", "subject": "Reduce log level of pgVersionFromStr() and pgVersionToStr()." }, { "commit": "596c62c54e375fd25e7e34e9604b47ecc1ab73ff", "date": "2023-01-14 15:25:25 +0700", "subject": "Simply return in pgVersionToStr()." }, { "commit": "34e4835ff30e7f44cb36b3f1ed29d670e74a75ca", "date": "2023-01-12 21:24:28 +0700", "subject": "Refactor common/ini module to remove callbacks and duplicated code.", "body": "The callbacks in iniLoad() made the downstream code more complicated than it needed to be so use an iterator model instead.\r\n\r\nCombine the two functions that were used to load the ini data to remove code duplication. In theory it would be nice to use iniValueNext() in the config/parse module rather than loading a KeyValue store but this would mean a big change to the parser, which does not seem worthwhile at this time." }, { "commit": "9ca492cecfd40516b8815080af23ee3e64efc22a", "date": "2023-01-12 17:36:57 +0700", "subject": "Audit mem contexts returned from functions into the calling context.", "body": "It is possible for functions to accidentally leak child contexts into the calling context, which may use a lot of memory depending on the use case and where it happens.\n\nUse the function return type to determine what should be returned and error when something else is returned. Add FUNCTION_AUDIT_*() macros to handle exceptions.\n\nThis checking is only performed during unit tests on the code being covered by the specific unit test.\n\nNote that this does not work yet for memory allocations, i.e. memNew(). These are pretty rare so are not as much of an issue and they can be added in the future." }, { "commit": "de1dfb66ca022e7ab8370503eacda94629f6ae3f", "date": "2023-01-12 17:14:36 +0700", "subject": "Refactor logging functions to never allocate memory.", "body": "Allocating memory made these functions simpler but it meant that memory was leaking into the calling context when logging was enabled. It is not clear that this was an issue but it seems that trace level logging could result it a lot of memory usage depending on the use case.\n\nThis also makes it possible to audit allocations returned to the calling context, which will be done in a followup commit.\n\nAlso rename objToLog() to objNameToLog() since it seemed logical to name the new function objToLog()." }, { "commit": "890b9d009390e472bd96ac74f6b347d7b1cc5a12", "date": "2023-01-12 15:04:49 +0700", "subject": "Remove mostly useless mem context in storageRemoteInfoProtocolPut().", "body": "Maybe this once had a deeper purpose but now at least it is just used to avoid a few trivial allocations. If we really wanted to do that a flag would be better but it does not seem worth the trouble." }, { "commit": "6ec2241e15c4d6d6c8fe9a09c2cd562631ba8423", "date": "2023-01-12 14:24:20 +0700", "subject": "Fix leak in protocolServer()." }, { "commit": "9d42435ffbaa20febae5a9a5c62446a0fe5d0090", "date": "2023-01-12 14:22:09 +0700", "subject": "Fix leaks in db module." }, { "commit": "7f7d515441ee28e10d5f2aa63f986128638342bc", "date": "2023-01-12 14:19:10 +0700", "subject": "Fix leak in configOptionRemote()." }, { "commit": "123d5e77ae188dfe75a431fcb131dcca1b1913bb", "date": "2023-01-12 14:17:48 +0700", "subject": "Minor optimizations on config/parse module." }, { "commit": "0becb6da3147e7992b7e91f78032a961e7c352be", "date": "2023-01-12 10:22:26 +0700", "subject": "Enhance libbacktrace to handle incomplete stack traces.", "body": "This fills in backtrace info at the bottom of the call stack when the stack trace is incomplete due to testing. This does not affect release builds, which is why it did not make the first cut, but it turns out to be useful for testing and barely changes the release code (when we do release this).\n\nThe recursion test in common/error was simplified because it would now return a very large trace." }, { "commit": "57fc4aaeb9b2aea84fce5bb813980116ad01dbe7", "date": "2023-01-12 09:06:01 +0700", "subject": "Update RHEL user guide to PostgreSQL 11/12.", "body": "PostgreSQL 10 has expired from the PGDG yum repository." }, { "commit": "74ca9b50bbdbb33dafedcc8e1230a83cb619c3f5", "date": "2023-01-11 19:43:10 +0700", "subject": "Revert macro changes from 38d32045.", "body": "This seemed like it would be cleaner but an important detail was missed (logAny) and it does not seem simpler when factoring that back in.\n\nKeep the removal of the extraneous semicolons and all the downstream changes required by the removal." }, { "commit": "9347313cbf1dbd5590b7ca481bd37bcb5de14b00", "date": "2023-01-11 19:16:31 +0700", "subject": "Fix error detail output to stderr.", "body": "The error detail should be output when the error is an assert (this part was working) or the log level is at least debug. In cases where log-level-console was at least debug but log-level-stderr was not the detail was lost.\n\nImprove the range checking to output error detail to stderr when log-level-console is at least debug." }, { "commit": "c8264291238e0bd199e4babab7013060e7fcba7d", "date": "2023-01-11 11:19:26 +0700", "subject": "Improve libbacktrace implementation.", "body": "The libbacktrace feature has not been working since the move to meson because libbacktrace detection was not added to the meson build. Add libbacktrace to meson and improve the feature so that it can be compiled into release builds.\n\nThe prior implementation fetched line numbers with each stack trace push. Not only was this slow but it missed any functions that were not being tracked on our stack.\n\nInstead just examine the backtrace when an error happens and merge it with the info we have on our stack. If the backtrace is not available then the output remains as before.\n\nAlso remove --backtrace from test.pl since the library is now auto-detected.\n\nLeave this library out of the production build for now to give it a little time to shake out in testing." }, { "commit": "0dd2997714ed31b4a005fc0061048f3697c9ea1a", "date": "2023-01-10 11:45:48 +0700", "subject": "Fix leak in jsonReadVarRecurse()." }, { "commit": "675b4ae251564742e23a9e6acdfb1d764b9b159d", "date": "2023-01-10 11:17:08 +0700", "subject": "Fix leak in archivePushDropWarning()." }, { "commit": "bf9e331ac15b0bd261f1a71e1307b51c9934aa85", "date": "2023-01-10 10:14:43 +0700", "subject": "Fix incorrect return type in backupFile()." }, { "commit": "b722d2b6ca31edead49b6dcc8fe0286bb508f5b2", "date": "2023-01-10 09:28:57 +0700", "subject": "Fix formatting of archiveGetFile()." }, { "commit": "38d320455336ab6da16c7c1b86e52d3118bcbd73", "date": "2023-01-09 14:47:57 +0700", "subject": "Remove duplication from FUNCTION_LOG_BEGIN_BASE() macro definitions.", "body": "It is a bit simpler to define STACK_TRACE_TEST_START()/STACK_TRACE_TEST_STOP() in a separate #ifdef so FUNCTION_LOG_BEGIN_BASE() does not need to be defined twice.\n\nAlso add missing semicolons exposed by this change." }, { "commit": "6633ccd18d66f3354ce6af27c93916bce330981b", "date": "2023-01-05 14:03:43 +0700", "subject": "Do not output stderr on unit/integration test errors.", "body": "Since stderr is being redirected to stdout this results in duplication of the error output." }, { "commit": "4429bc82f530fd87cd4e370c012054be117b76b9", "date": "2023-01-05 12:59:06 +0700", "subject": "Add unit tests for the unit test build code.", "body": "When this code was migrated to C the unit tests were not included because there were more important priorities at the time.\n\nThis also requires some adjustments to coverage because of the new code location." }, { "commit": "a5499afabc7729efc2254010b47aeced74d0e918", "date": "2023-01-05 12:44:00 +0700", "subject": "Add CHECK_FMT() macros and tests for CHECK*() macros.", "body": "The CHECK_FMT() macro will be used in some upcoming tests but could also be useful in the core code.\n\nThe CHECK*() macros never had tests so add them." }, { "commit": "2e3513984ef528404341e4e399f5257d6f2f1ccb", "date": "2023-01-05 12:33:17 +0700", "subject": "Remove extra linefeed." }, { "commit": "877bb2ac9e452ddeef6ddd4845c4fb59d83379fe", "date": "2023-01-03 08:26:44 +0700", "subject": "Update LICENSE.txt and PostgreSQL copyright for 2023." }, { "commit": "2b2a6edf355f50c0d399243e6d4d9fa495a1ec34", "date": "2023-01-02 18:53:05 +0700", "subject": "Cleanup of common/ini module in preparation for a refactor." }, { "commit": "92d5fd17b4ddb2e72e4cf5c10fd7708d5a9fad56", "date": "2023-01-02 17:32:09 +0700", "subject": "Fix formatting of restoreFile() definition." }, { "commit": "d51a86c6210001359f86aadc36285137fe8f2da9", "date": "2023-01-02 17:30:57 +0700", "subject": "Add BUFFER_EXTERN() and cleanup hash zero constants.", "body": "BUFFER_EXTERN() provides a clean way to create buffer constants.\n\nConvert HASH_TYPE_SHA256_ZERO_STR to HASH_TYPE_SHA256_ZERO_BUF to be consistent with HASH_TYPE_SHA1_ZERO_BUF." }, { "commit": "66f108ea8a01ac881a47f4cbd8d5fb3ca8102e1d", "date": "2023-01-02 17:26:53 +0700", "subject": "Cleanup terminology in Buffer/Variant/String macros." }, { "commit": "f0189129088456cf22f7bf86fa505cf9c28a5e37", "date": "2023-01-02 15:24:51 +0700", "subject": "Split VR_EXTERN/FN_EXTERN macros from FV_EXTERN.", "body": "This should make it a little clearer what the variable (VR) macros are doing since the declaration/definition cannot both be set to extern (but functions can).\n\nSplitting the variable macros out also allows them to be changed in the future with little churn, while changing the function macro creates a large amount of churn." }, { "commit": "9e29c01891225dd5f371b76d324366bbb3694ca4", "date": "2023-01-02 15:11:37 +0700", "subject": "Remove MacOS from CirrusCI.", "body": "CirrusCI stopped supporting Intel but the arm builds are not working, even with the same steps that work on an arm Mac.\n\nRemove to unstick the build pipeline until this can be resolved." }, { "commit": "8993267df9864852efd59a9b5366e9a521a6c08e", "date": "2022-12-31 23:33:45 +0700", "subject": "Fix formatting." }, { "commit": "1793839e5a821a7177922e4cdad104eba772c3ec", "date": "2022-12-31 19:12:35 +0700", "subject": "Clean up storage headers and move posix internal type." }, { "commit": "4fb8a0ecdd12bdb01273cebbecf61f6beb22146d", "date": "2022-12-31 17:13:41 +0700", "subject": "Add meson unity build and tests.", "body": "This is immediately useful because it will detect any extern'd functions or variables that are not being used. It also detects functions or variables that are declared but not defined.\n\nIf a FV/VR_EXTERN macro is missing it will be detected either because of a mismatch in the declaration/definition or because a new defined symbol will appear in the nm test.\n\nEventually the unity build will be used to create a more optimized pgbackrest binary but that will need to wait." }, { "commit": "8aa2b101bbd9fe07971794bf4bb4c8d2a61c7b07", "date": "2022-12-31 16:41:01 +0700", "subject": "Put logging functions unused by release builds into DEBUG blocks.", "body": "Also inline some functions that are needed in non-DEBUG builds." }, { "commit": "4a64c5d80c9af2a3498943f326de7295679c63e6", "date": "2022-12-31 15:49:32 +0700", "subject": "Use memcpy() instead of strncpy when source size is known.", "body": "In this case the destination will be large enough to hold the source so memcpy is more efficient.\n\nAlso, in highly optimized builds the compiler may warn for strncpy() when it can see that the source size won't include the terminator." }, { "commit": "2332ce8ffc16494766f1f6ee07341242e2b7b7c2", "date": "2022-12-31 13:14:27 +0700", "subject": "Move storageHelperFree() to storageHelper test harness.", "body": "This function was only used for testing so move into a test harness." }, { "commit": "8b218158ae70136d7a2ddc5a7347f79e69fbeb13", "date": "2022-12-31 12:54:33 +0700", "subject": "Move regExpMatchPtr()/regExpMatchStr() to build/common/regExp module.", "body": "Similar to b9be4fa5, these functions are not used by the core code so move them to the build module. The new implementation is a little less efficient but that is much less of a worry in the build/test code.\n\nAlso remove regExpMatchSize() since it was not longer needed." }, { "commit": "45ece13678f94c57777180710a7ea71a1c60971a", "date": "2022-12-31 11:57:06 +0700", "subject": "Use error*() getters where possible.", "body": "This makes it a little easier to change the underlying representation and ensures that the functions are used." }, { "commit": "fa9d831f9f722f28567c4997551cd6bd1d7077d3", "date": "2022-12-31 11:09:50 +0700", "subject": "Move xmlNodeAttribute() to build/common/xml module.", "body": "Similar to b9be4fa5, this function was not used by the core code so move it to the build module." }, { "commit": "c6264ad4cf55920558513127d4e221ce8e85eb28", "date": "2022-12-31 10:31:15 +0700", "subject": "Fix comment typo for strLower()." }, { "commit": "163a004f30675403eba2a619412bd803eba4d71a", "date": "2022-12-31 10:26:11 +0700", "subject": "Move strReplace() and strUpper() to build/common/string module.", "body": "Neither of these functions were used by the core code. strReplace() is only used in the tests but it doesn't hurt to put it in build since the build code is not distributed." }, { "commit": "d517d4a3283c0ccc5ca5d06593d1b244dd5e68b4", "date": "2022-12-31 10:10:44 +0700", "subject": "Add explicit keyword for covered modules included in other modules.", "body": "This was done by checking the extension but it is possible to include a module that does not have a vendor or auto extension. Instead make it explicit that the module is included in another module.\n\nAlso change the variable from \"include\" to \"included\" to make it clearer what it indicates." }, { "commit": "729d7f071dafe22f85201bfaf2bae461ec54d285", "date": "2022-12-30 18:45:11 +0700", "subject": "Put memContextSize() in a DEBUG block.", "body": "This function is only used in unit tests so no need to include it in the release build." }, { "commit": "cebbf0d012ad7204c4770efa3308daddc267415f", "date": "2022-12-30 16:26:48 +0700", "subject": "Remove unused functions.", "body": "These functions were either added with the intention that they would be used or they became obsolete over time." }, { "commit": "6f0c652e2c50f288fc4f541a994281032856be2b", "date": "2022-12-30 14:59:16 +0700", "subject": "Remove declaration for function that is no longer defined." }, { "commit": "416fadd60cc0401e5e730867d98ae59ff54f8dcd", "date": "2022-12-30 14:48:31 +0700", "subject": "Mark functions not used externally as static.", "body": "Also cleanup a stray linefeed." }, { "commit": "b9be4fa54039e2c79fad0dc1552c7ffaf883004c", "date": "2022-12-29 15:37:27 +0700", "subject": "Restore errors when no backup matches the current version of PostgreSQL.", "body": "It is probably not a good idea to restore the latest backup when it was not made from the current PostgreSQL version. If there is no backup after a stanza-upgrade then replicas might be built with a prior version leading to failures.\r\n\r\nAdd an error in this case if the latest backup would be used, i.e. --set or --type=time/lsn is not specified." }, { "commit": "aa1e72dfe662678919d3b01aa0e6454b677873ad", "date": "2022-12-29 15:05:45 +0700", "subject": "Fix improper formatting in config/parse module.", "body": "The prior formatting worked but was confusing to read." }, { "commit": "36ee30d118ce6cc7720653a3d3a49a06c655c463", "date": "2022-12-28 19:15:44 +0700", "subject": "Updates and clarifications to index page.", "body": "In particular the section about other backup solutions not supporting parallel processing was no longer accurate, so reword it.\r\n\r\nAlso update some other sections that used older nomenclature, had awkward wording, or needed clarification." }, { "commit": "84a3ff8b7ac1243c0b47c6585697f4fcf3c12065", "date": "2022-12-28 10:48:44 +0700", "subject": "Clarify target-action behavior on various PostgreSQL versions.", "body": "The behavior of pause depends on the hot_standby parameter and the PostgreSQL version so mention both.\r\n\r\nThis behavior has been verified on PostgreSQL 9.6–15. PostgreSQL 12 is an inflection point because the behavior of an unset recovery_target_action with hot_standby=off changed in https://git.postgresql.org/gitweb/?p=postgresql.git;h=2dedf4d9a899b36d1a8ed29be5efbd1b31a8fe85." }, { "commit": "ae258f604e634f1d2a760ffdb92dd359fd1129bb", "date": "2022-12-27 20:28:38 +0700", "subject": "Add replacement for linefeeds in monitoring example.", "body": "The copy command was converting \\n to a linefeed, which the json conversion did not like. In a healthy repository there won't be any linefeeds but certain errors can contain them.\r\n\r\nFix by loading into a text field and then replacing the linefeed when converting to jsonb." }, { "commit": "44da314adb2b63f96b1d571f822e91a0e412e984", "date": "2022-12-27 20:05:08 +0700", "subject": "Add compress-level range checking for each compress-type.", "body": "The prior range checking was done based on the valid values for gz. While this worked it was a subset of what is available for lz4 and zst.\r\n\r\nAllow the range to be specified for each compress-type. Adding this functionality to the parse module would be a better solution but that is a bigger project than this fix deserves, at least for now." }, { "commit": "34110cfa0134d181c749cc040446f703b643bc9c", "date": "2022-12-27 11:31:59 +0700", "subject": "Fix typo." }, { "commit": "0b67389185558d4593be1ca56143de6a43e45f9f", "date": "2022-12-27 09:18:17 +0700", "subject": "Improve comment on forced WAL switch at backup start." }, { "commit": "a63e87ea2cd0efbf9e9b78adccdf7285a6876b3b", "date": "2022-12-23 14:06:35 +0700", "subject": "Move function prototype to a better location in the header." }, { "commit": "56b55f81e894c0b2875f92913bda8f0679009b17", "date": "2022-12-22 09:26:26 +0700", "subject": "Add repository checksum to make verify and resume more efficient.", "body": "Calculate a checksum of the data stored in the repository when a file is transformed (e.g. compressed). This allows resume and verify to operate without needing to decompress/decrypt the data.\r\n\r\nThis can also be used to verify more complex formats such as block incremental and allow backups from the repository without needing to decompress the data to verify the checksum.\r\n\r\nAdd some basic encrypted tests to maintain coverage. These will be expanded in a future commit." }, { "commit": "2ab845e2636fcd865032075b4dce9e520d094d87", "date": "2022-12-20 16:35:27 +0700", "subject": "Store manifest checksums in memory more efficiently.", "body": "Manifest checksums were stored as hex-encoded strings due to legacy compatibility with Perl. Storing the checksums as binary in memory uses half the space and avoids many conversions.\r\n\r\nThere is no change to the on-disk manifest format which stores the checksum as a hex-encoded string." }, { "commit": "77c721eb635c6d757d2d5ed6f2ffc083720c10a7", "date": "2022-12-20 12:20:47 +0700", "subject": "Remove support for PostgreSQL 9.0/9.1/9.2.", "body": "Our new policy is to support ten versions of PostgreSQL, the five supported releases and the last five EOL releases. As of PostgreSQL 15, that means 9.0/9.1/9.2 are no longer supported by pgBackRest.\r\n\r\nRemove all logic associated with 9.0/9.1/9.2 and update the tests.\r\n\r\nDocument the new support policy.\r\n\r\nUpdate InfoPg to read/write control versions for the history in backup.info, since we can no longer rely on the mappings being available. In theory this could have been an issue after removing 8.3/8.4 if anybody was using a version that old." }, { "commit": "17ce738ff749c50f4ca721144cd9addce9e895f2", "date": "2022-12-11 20:05:31 +0700", "subject": "Use smaller type for decode tables.", "body": "This saves a bit of space in the binary and should not have a noticeable impact on performance." }, { "commit": "010efffb0c3c7935f789e5c560104e5a3019a28f", "date": "2022-12-11 19:46:48 +0700", "subject": "Add hex encode/decoding to decode module.", "body": "This replaces the bufHex() function and also allows hex to be decoded." }, { "commit": "9a9ee8e6406721c5a1825b8a25baef944802c92d", "date": "2022-12-11 18:55:25 +0700", "subject": "Rename EncodeType enum to EncodingType.", "body": "This avoids constructs such as decodeToBin(encodeBase64, ...) which are confusing since decode and encode are used in the same function call. decodeToBin(encodingBase64, ...) makes it clearer what is happening." }, { "commit": "1345caa3f70e24ff759967692c10c07e63f44d00", "date": "2022-12-07 20:30:38 +0700", "subject": "Update missing flag in storage/posix test.", "body": "The storageNewItrP() permissions test was running twice with the errorOnMissing flag set to false. Fix by setting to true for one test.\n\nAlso update the comments to be clearer about what the tests are doing and fix minor formatting." }, { "commit": "81d84ab495bfeaf0102adce2425ab3a528bb8845", "date": "2022-12-07 10:57:29 +0100", "subject": "Fix typos." }, { "commit": "4dc632d57064420134f3a198fa18dc951d5bf574", "date": "2022-12-05 14:15:15 +0800", "subject": "Add backup test harness.", "body": "This allows test backups to be run in other test modules.\n\nIt is likely that more logic will be moved here but for now this suffices to get test backups working in the restore module." }, { "commit": "96cf479d3c46b310b87e69bd2e19a30e71e5824c", "date": "2022-12-05 14:00:25 +0800", "subject": "Add missing header in lock module." }, { "commit": "c972a9359beb5b75c4b0ea57bcb12a34fe5e7d84", "date": "2022-11-28 17:56:59 +0800", "subject": "Begin v2.44 development." }, { "commit": "cc2ffd8264bb4e10e6b289ce3978452018f725e8", "date": "2022-11-28 17:47:48 +0800", "subject": "v2.43: Bug Fix" }, { "commit": "c4bf775099c85e3cd081f9c1ce35efd4c71d9161", "date": "2022-11-28 16:42:35 +0800", "subject": "Fix missing reference in diff/incr backup.", "body": "When loading prior manifests without the new reference list, the code failed to add the current backup to the reference list. Since the current backup is never explicitly referenced, building references from the file list was not sufficient to generate a complete list.\r\n\r\nThe main problem here was a bad test, fixed in 28f6604. This masked the issue and prevented it from being found. Now it is clear in the test that the current label is missing from the reference list.\r\n\r\nFix by adding the current label to the reference list if a reference list is not stored in the manifest." }, { "commit": "28f660482aa5abb97850a9d7c89c2e5e9f23a4c9", "date": "2022-11-26 10:48:02 +0800", "subject": "Move manifestBackupLabelSet() test in info/manifest unit.", "body": "Changing the label of a manifest that already had a label was not a good test and it ended up masking a bug where the current backup label was not being added to the reference list on manifest load, since manifestBackupLabelSet() added the label to the reference list. In fact, manifestBackupLabelSet() should never be called after a manifest load or even after the label has been set.\n\nAdd an assertion to prevent manifestBackupLabelSet() being called when the label is already set.\n\nThe bug exposed here will be fixed in a subsequent commit." }, { "commit": "3f363cb3aed1a7c6836ee6c32eae600129a6f6b8", "date": "2022-11-22 15:04:13 +0800", "subject": "Add hint when an option is specified without an index.", "body": "Hopefully this will make it a little clearer to the user what is wrong when they specify an indexed option without an index.\r\n\r\nAlso fix an ambiguous use of cfgParseOptionP(). The prior code worked in that it set prefixMatch = true but it was not very readable." }, { "commit": "27460862e77b7fba327c92448b2ec8ad41f2016c", "date": "2022-11-22 10:29:32 +0800", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "092e254794c6d9eba9fc3175357bc91257f2eedf", "date": "2022-11-22 10:27:20 +0800", "subject": "Begin v2.43 development." }, { "commit": "70b75532bf20a025323b2a3138497ef01eb1774f", "date": "2022-11-22 10:20:59 +0800", "subject": "v2.42: Bug Fixes" }, { "commit": "3ad588443b452e04f3abcc107e825dc47e078103", "date": "2022-11-14 12:47:27 +0800", "subject": "Update test.pl --psql-bin option to match command-line help.", "body": "The option to specify the path to psql was shown in the command-line help as --psql-bin but the option was actually named --pgsql-bin.\r\n\r\nRename to match the help so they are consistent." }, { "commit": "58b3c91babc6af9e003e5ef09bf99becf8a6f827", "date": "2022-11-10 10:28:49 +0930", "subject": "Add raw mode to CipherBlock to save space.", "body": "The magic in the header is only required so that command-line openssl will recognize the file as being encrypted. In cases where the encrypted data cannot be read with the command-line tool it makes sense to omit the header magic to save some space.\n\nUnfortunately this cannot be enabled for file bundling because it would break backward compatibility. However, it should be possible to enable it for the combination of bundling and block incremental." }, { "commit": "c9db7bc274211820030c4ab6ab8c70c1b1896254", "date": "2022-11-06 16:12:23 +0930", "subject": "Update cipherBlockNew() to allow optional parameters.", "body": "This simplifies calls a bit since digest is never passed and allows for new optional parameters." }, { "commit": "221db610d2a054ed00ad27377663941abfb9fb98", "date": "2022-10-18 18:02:17 +1300", "subject": "Shorten names in real/all integration test matrix.", "body": "This should allow one or two more parameters to be added without going to a new line, which keeps the matrix easier to read." }, { "commit": "5fceee88a9439460bf24082f275ec2e695cc6bf6", "date": "2022-10-18 17:39:59 +1300", "subject": "Add backupFileRepoPathP().", "body": "The path for a backup file in the repository was being generated in four different places, so move the logic to a function." }, { "commit": "fee38c2c7cc3ae318bf0d3b50fe23c22900c2311", "date": "2022-10-18 16:11:35 +1300", "subject": "Pass filters to remote storage as a handler array.", "body": "The prior code required coverage in the storage/remote module for all filters that could be used remotely.\n\nNow the filter handlers are set at runtime so any filter list can be used with a remote. This is more flexible and makes coverage testing easier. It also resolves a test dependency.\n\nMove the command/remote unit test near the end so it will have access to all filters without using depends." }, { "commit": "1730ef4ac3d88137bd218e6128b0107a8bf2c121", "date": "2022-10-18 11:33:19 +1300", "subject": "Add noTruncate flag to storageNewWriteP().", "body": "This flag skips truncation when opening a file for write on drivers that support it, currently Posix and CIFS. This is convenient for cases where the file needs to be manipulated directly using the file descriptor. Using the file descriptor is not ideal and additional functionality should be added to the storage interface, but for now at least this avoids code duplication, especially on close which updates owners, the timestamp, syncs, etc.\r\n\r\nThe remote driver forbids no truncate because a file descriptor is never available for a remote storage write object.\r\n\r\nUpdate two instances in the current code which benefit from this new functionality, but the primary reason for the change is to support more complex restore deltas in the upcoming block incremental feature." }, { "commit": "7967c750d83466d030081e9deb1d70a1ba240271", "date": "2022-10-18 11:09:00 +1300", "subject": "Fix protocol error on short read of remote file.", "body": "If a remote file read was stopped before the read was complete or if an error occurred in the middle of the read then the protocol would end up in a bad state and produce this error:\r\n\r\nProtocolError: client state is 'data-get' but expected 'idle'\r\n\r\nPrevent this by reading the rest of the file on close() or free() to leave the protocol in an idle state for the next command.\r\n\r\nThis was a possible issue for bundling because the amount to read is known in advance and therefore eof may not be reached. However, I was only able to reproduce this issue with unreleased code.\r\n\r\nOn error this issue would cause the original error to be lost. The process may still fail with this fix (if the error comes from another source) but hopefully we'll get better information about the original error." }, { "commit": "ddd966cadc30440a81100144204d4084a0f2ac3a", "date": "2022-10-18 09:59:15 +1300", "subject": "Use more generic descriptions for pg_start_backup()/pg_stop_backup().", "body": "The names were changed in PostgreSQL 15, so update the code and docs to make the naming more generic where needed to avoid using a version-specific name in the logs and documentation." }, { "commit": "65be4c64a984d51306db4c6febe3a124c3230f51", "date": "2022-10-16 09:58:35 +1300", "subject": "Finalize catalog number for PostgreSQL 15 release." }, { "commit": "ea162e821634919b7c839e866b99bf82010dbacc", "date": "2022-10-14 12:54:43 +1300", "subject": "Move storageReadRemoteOpen() in storage/remote/read module.", "body": "This will make an upcoming important fix easier to review." }, { "commit": "909be412f8ad0d87f23ce2a2dbca94c3e0732f9f", "date": "2022-10-14 12:08:40 +1300", "subject": "Swap command/backup and command/restore unit tests.", "body": "Logically restore belongs after backup and in a future commit restore will have a dependency on some backup objects." }, { "commit": "8f67fb6db2645c2dae24429555ead66147ba176f", "date": "2022-10-14 10:59:07 +1300", "subject": "Enable FreeBSD 12/13 builds on Cirrus CI.", "body": "The builds were disabled in bb11539a and 164548f2 due to an error that seems to have been caused by a bad package dependency for rsync. In any case adding this fixed it:\n\npkg update && pkg upgrade -y libiconv" }, { "commit": "e7e106f781be01027d4829b258902549eb36eb7b", "date": "2022-10-11 10:52:34 -1100", "subject": "Use read-only storage to calculate checksum in restoreFile().", "body": "Writable storage is not required here so use read-only storage instead." }, { "commit": "164548f2b744db5e6c7a3089176f6df2a638d105", "date": "2022-10-11 10:20:44 -1100", "subject": "Disable FreeBSD 12 builds on Cirrus CI.", "body": "Build have begin failing with this error:\n\nld-elf.so.1: /usr/local/bin/rsync: Undefined symbol \"locale_charset\"\n\nThere does not appear to be a new version so hopefully this is a transient error (hoping the same for FreeBSD 13, see bb11539a). Disable for now to free the build pipeline." }, { "commit": "352cbf1d5738a720292b5067051a05efc2b9a8ac", "date": "2022-10-06 10:15:38 -1000", "subject": "Update ManifestFile booleans to bit fields." }, { "commit": "46a0af353139f3238dbe96671e99ea0456028940", "date": "2022-10-05 18:05:56 -1000", "subject": "Return binary as result from CryptoHash filter.", "body": "The prior result was hex-encoded, which is not optimal. This was legacy from the interface with Perl and then the JSON protocol. The new binary protocol natively supports binary so it makes sense to use it and convert to hex where needed.\n\nA number of these hex conversions can now be removed but that will need to be handled in another commit." }, { "commit": "5602f179a177a8513b96c762590829c012c28175", "date": "2022-10-05 17:01:35 -1000", "subject": "Add varint-128 encode/decode to IoRead/IoWrite.", "body": "This makes it more efficient to read/write (especially read) varint-128 to/from IO.\r\n\r\nUpdate the Pack type to take advantage of the more efficient read and remove some duplicate code." }, { "commit": "102ce5dee47e8d9a8d2eeaf2b2cc2bac105914ef", "date": "2022-10-05 16:28:31 -1000", "subject": "Add persistent reference list to manifest.", "body": "The reference list was previously built at load time from whichever references existed in the file list. This was sufficient since the list was for informational purposes only.\r\n\r\nThe block incremental feature will require a reference list that contains all prior backups, even those that are not explicitly referenced from the manifest. Therefore it makes sense to build and persist a manifest list rather than building it at load time.\r\n\r\nThis list can still be used for informational purposes, though it needs to be sorted since the list it sill built for older manifest versions and may not be in sorted order.\r\n\r\nAdd strLstFindIdx() to find references in the list." }, { "commit": "c647bcb50954f623d29104cdb8a1457ed8e94579", "date": "2022-10-05 13:14:15 -1000", "subject": "Add manifest flags for file processing during backup.", "body": "The prior method was to check a combination of fields to determine if a file needed to be copied, delta'd, or resumed. This was complicated and ultimately imposed a limitation on the number of operations that could be performed.\r\n\r\nIntroduce copy, delta, and resume flags in the manifest to make it clearer which operations need to be performed and to reduce complex and duplicated logic.\r\n\r\nThis also allows zero-length bundled files to be completed during manifest build rather than later on during backup processing." }, { "commit": "bb11539a37e830ed2a836633218b7184a7a57f2d", "date": "2022-10-05 12:20:02 -1000", "subject": "Disable FreeBSD 13 builds on Cirrus CI.", "body": "Build have begin failing with this error:\n\nld-elf.so.1: /usr/local/bin/rsync: Undefined symbol \"locale_charset\"\n\nThere does not appear to be a new version so hopefully this is a transient error. Disable for now to free the build pipeline." }, { "commit": "1ea6a4142e47e2e0b56911ad1f028dfcddc5fd64", "date": "2022-10-04 14:19:12 -1000", "subject": "Improve manifest file updates.", "body": "The prior manifestFileUpdate() function was pretty difficult to use since all the parameters had to specified. Instead, pass a ManifestFile struct that has all members set as needed.\r\n\r\nWhen new struct members are added the manifestFileUpdate() call sites will still need to be reviewed, but this should make the process of adding members a bit simpler." }, { "commit": "f981fb45d9931e597afec764e13b1a53d8365d25", "date": "2022-10-04 13:22:31 -1000", "subject": "Do not store references for zero-length files when bundling.", "body": "This appears to have been an oversight in 34d6495. Storing the reference is not really correct since the file is not stored in a prior backup. It also uses more space.\r\n\r\nThere is no real harm in storing the reference, since it is always ignored on restore, but the code is simpler if the zero-length files can be dealt with during the manifest and don't need additional handling later on. This is also an important part of some upcoming optimizations." }, { "commit": "f0acc195c0527495857f1afa8b24bdf2a4f477b5", "date": "2022-10-03 10:02:44 -1000", "subject": "Fix assert message and add a clarifying comment." }, { "commit": "6e26860c2648f6e23ad99bb4d38ecb1bfc45dd0c", "date": "2022-10-02 17:48:43 -1000", "subject": "Do not log bundle info when a file is delta matched from a prior backup.", "body": "It is possible to log the bundle info correctly but the information is useless with the backup reference, which does not appear until later. For now just omit the bundle info so we are not logging something incorrect." }, { "commit": "4722ad87a78f1085ef8d823b7460d191477e768e", "date": "2022-10-02 17:41:31 -1000", "subject": "Add test for differential file bundles to the command/backup unit.", "body": "This test exposes a small logging issue. The bundle information for the matched delta on PG_VERSION is not correct. This issue will be fixed in the next commit.\n\nThe information stored in the manifest *is* correct so this bug is essentially cosmetic." }, { "commit": "ac99201c0ebde616279214e600e49fdf506f7469", "date": "2022-10-02 17:32:48 -1000", "subject": "Add bufferSize to cvtUInt64FromVarInt128() to further limit reads.", "body": "The current call site, manifestFileUnpack(), does not know the total buffer size but the buffer has always been maintained in memory so there should be no corruption. However, there are upcoming use cases where the buffer will be read from IO, the buffer size will be known, and additional sanity checking on buffer overruns will be valuable.\n\nAlso rename params to align better with cvtUInt64ToVarInt128()." }, { "commit": "01b81f9d374701eb489dd1888bcba25fb5d6df84", "date": "2022-10-01 15:26:44 -1000", "subject": "Move link creation to storage interface.", "body": "Direct link creation via Posix functions has been moved to the Posix driver.\r\n\r\nThis change allows adding SFTP softlink creation in the SFTP driver using the standard interface." }, { "commit": "2a4137ed2eeaa769508301056226d6f86a67fd06", "date": "2022-09-25 16:15:23 -0800", "subject": "Add zero-length chunked content test to common/ioHttp unit.", "body": "It seems wasteful to chunk content when there is nothing to send but make sure we handle it gracefully just in case a server decides to do it." }, { "commit": "64b64b614c8272d2a87ffcde2fa7f27cf7eee365", "date": "2022-09-25 12:30:30 -0800", "subject": "Fix comment typo." }, { "commit": "f1e8e49fa933dd0f5dfab446f511ac010eb4d236", "date": "2022-09-23 14:00:58 -0700", "subject": "Use large error/log buffers in test harness.", "body": "Ninja produces quite a bit of output so error messages are often truncated by the default error/log buffers. Use large buffers in the test harness to capture the error even when there is a lot of output.\n\nNinja has introduced a --quiet option, but it is currently too new to be in any of our test distributions." }, { "commit": "cd8db7d9e53e041ad6fb31f2f015541c1d24abc2", "date": "2022-09-22 22:42:01 -0700", "subject": "Fix memory leak in file bundle backup/restore.", "body": "When converting restoreFile() to support file bundling in 34d64957 there were some I/O objects that were only freed at the end of the function that should have been freed at the end of each loop. Wrap the loops in temp mem contexts to fix this.\r\n\r\nDo the same to backupFile() since it would have a similar leak when resuming a backup. Since file bundles cannot be resumed the leak would not be as severe, but still seems worth doing to protect against future leaks." }, { "commit": "d50a4442e45efa8d1a16c9bf68f2337367c42835", "date": "2022-09-22 10:35:41 -0700", "subject": "Add missed release note for b05d31f5." }, { "commit": "9483844f7fc9a5eac64a87d3c80cfea3fd592e4e", "date": "2022-09-19 10:25:38 -0700", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "ab4209ebcbe23b5c3b759ba67f0e9bdc6cb66280", "date": "2022-09-19 10:17:25 -0700", "subject": "Begin v2.42 development." }, { "commit": "6b355806cc288e6f473254412337596287291798", "date": "2022-09-19 10:08:10 -0700", "subject": "v2.41: Backup Annotations" }, { "commit": "c39c9f220ed328b4175d912e109697b726e9ea9b", "date": "2022-09-15 12:00:44 -0700", "subject": "Fix issue when listing directories recursively with a filter.", "body": "While recursing and filtering, if the last entry in a directory was another directory containing entries then the parent list would get freed too early, causing a double free error or segfault.\r\n\r\nFix by ensuring that the completed list is at the top of the stack before freeing it. This will defer freeing parent lists until the contents of paths have been processed." }, { "commit": "08a44be4c38e4bc72a20c19d764be8a673c9b909", "date": "2022-09-14 16:22:48 -0700", "subject": "Fix incorrect storage for rendering errors.", "body": "Coverity complained of a copy/paste error here, but the actual error was what it took to be the correct \"copied from\" code.\n\nIn any case, the prior code would have blown up as soon as a new error type was added. Fix by updating to the writable build storage." }, { "commit": "240cd755d19ae2688d0deee65c00daf0651bb601", "date": "2022-09-14 10:06:06 -0700", "subject": "Add mem context test missing from 0f7b6a33.", "body": "A coverage exception was added during development but was not removed before commit.\n\nRemove the exception and add a test for coverage." }, { "commit": "8fb61a809dc63936a78e5233a2e8bc8dc616c762", "date": "2022-09-08 18:36:03 -0600", "subject": "Add FN_INLINE_ALWAYS macro.", "body": "Eliminate a lot of useless repetition for a commonly used pattern." }, { "commit": "2c9cbbcc4e72f5c1809e0001e40bfb3e7d8bbe01", "date": "2022-09-08 18:16:38 -0600", "subject": "Update Cirrus CI FreedBSD 13 image version." }, { "commit": "3b5df1e089234d425dd225f8c1976755dc947c8a", "date": "2022-09-02 10:03:18 -0500", "subject": "Update archive.info timestamps after a successful backup.", "body": "Lifecycle policies can cause the archive.info file and its copy to be removed since they are only updated on a stanza-upgrade. Update the timestamps after a successful backup to prevent this.\r\n\r\nThis does not mean that lifecycle policies should be used as a replacement for expiration. However, in some cases there may be policies in place that are out of admin control. If the lifecycle expiration is less than pgbackrest expiration then corruption of the earliest backup will occur at the very least and there might be other corruption which would make the repo unrecoverable." }, { "commit": "0f7b6a3344b676d79afa7cd16ab54f67be870fdf", "date": "2022-08-31 12:44:55 -0400", "subject": "Skip mem context cleanup in CATCH_FATAL() block.", "body": "An error that gets raised all the way to the top TRY block might need to free a lot of resources and any of these callbacks could throw an error and mask the original error. In fact this is pretty likely since we are already in an error state. For example, the Db object will try to close the remote db connection, but if the protocol is in a bad state it will not be able to do so.\r\n\r\nSolve this, for now, by not freeing memory or calling callbacks in the CATCH_FATAL() block. This gives us a better chance if being able to report the error without encountering another error first.\r\n\r\nFor the most part, we don't need to worry about freeing resources (file handles, TLS contexts, etc.) if the program is going to exit immediately. However, it is important to attempt to terminate all active protocol connections, which is done by protocolFree() in main() since the protocol objects live in the top context.\r\n\r\nAnother way to handle this would be to implement an error stack and that is probably something we will do in the future. But, in the case of a segfault the original error would still be lost. Yet another option would be to still do cleanup but defer it until after the CATCH_FATAL() block." }, { "commit": "eda7f81ee4fd6818ce458768144fc3fc1e1f0f21", "date": "2022-08-31 10:01:12 -0400", "subject": "Fix incorrect time expiration being used for non-default repositories.", "body": "If a repo is not specified for the expire command then the lowest repo becomes the default. The repo-retention-full value for time was being retrieved from the default rather than a specific repo which led to an incorrect expiration being applied.\r\n\r\nGet the value from the specific repo and add a test.\r\n\r\nIt would be better if the default repo could not be queried in this case but it is not clear how to do that since the repo option is valid for expire (unlike, e.g., archive-push)." }, { "commit": "f1cb8485917bc07e00e77efe4593fb737f72136a", "date": "2022-08-30 18:04:32 -0400", "subject": "Fix comment typos in command/expire unit test." }, { "commit": "db75ffd27037d690a11eb02902f929250fe2bf5b", "date": "2022-08-25 10:12:22 -0400", "subject": "Support --set in JSON output for info command.", "body": "Allow detailed information (e.g. error list, tablespace list) in JSON output that is already available in text output with the --set option." }, { "commit": "381fd0a5a4de61624615b150ba98beeafac31f7e", "date": "2022-08-24 10:52:33 -0400", "subject": "Backup key/value annotations.", "body": "Allow key/value annotations to be added with the backup command and added/modified/removed with the new annotate command.\r\n\r\nAnnotations can be viewed with the info command in text mode when --set is specified and are always included in JSON output." }, { "commit": "b05d31f531b55b747175cf94c963f95574c54d2b", "date": "2022-08-24 09:33:26 -0400", "subject": "Allow upload chunk size to be configured for object stores.", "body": "There are performance benefits to increasing the upload chunk size as long as the tradeoff with additional memory consumption is acceptable.\r\n\r\nMake the chunk size configurable for S3, GCS, and Azure, but don't attempt to do any validation of the chunk size beyond some sane limits. The defaults remain as is for each storage type to avoid any unintentional regressions." }, { "commit": "37b4592e522289a0523f03f8e9b144ff28fa21d3", "date": "2022-08-24 08:45:44 -0400", "subject": "Allow host memory limits in the user guide to be disabled.", "body": "These limits can cause errors in some environments, e.g. Docker in Docker on Mac M1.\n\nEntirely remove limits from the build, s3, and azure hosts since memory usage on these hosts is out of our control and not useful for testing.\n\nAlso allow empty variables to be rendered as blank." }, { "commit": "ff1188f92dfc9e8f6d4aeca8b71a9886f24f58b4", "date": "2022-08-22 13:51:05 -0400", "subject": "Allow quote tag in command-line help.", "body": "The quote tag will be used in an upcoming commit." }, { "commit": "2a99ac4324b52b8b42db639e645378840159af00", "date": "2022-08-17 11:24:20 -0400", "subject": "Clear ProtocolClient callback after connection error.", "body": "Attempting to shut down the connection will fail since the server has already disconnected and a new error will be thrown, masking the original error." }, { "commit": "82786da1545c0bfe8185837a35d13a50b5e11ade", "date": "2022-08-16 16:15:48 -0400", "subject": "Do not allow CATCH() to catch a fatal error.", "body": "Catching individual fatal errors was only used in testing so the tests have been updated to use other errors instead. CATCH_FATAL() is now the only way to catch fatal errors.\r\n\r\nThis simplifies the logic a bit for upcoming changes to error handling and cleanup.\r\n\r\nAlso fix an issue where passing errorMessage() directly to THROW*() would attempt to copy the message buffer instead of preserving it, which is undefined behavior. Since there were no instances of this behavior before this commit, this was not a live bug." }, { "commit": "02665a5894dc4b025172b32c7a1ce42c81d6c582", "date": "2022-08-02 11:05:31 -0400", "subject": "Update Minio test/documentation container version." }, { "commit": "eb287b18c882a16eecfb1445a513c4da4ccab1b7", "date": "2022-07-29 10:31:36 -0400", "subject": "Add profiling, performance, and optimization to C test harness.", "body": "All unit and performance tests are now built by the C harness.\n\nRemove all unit/performance test build code from Perl.\n\nRemove code from C harness that is no longer used. This code was included so the C harness could be run separately, but that is no longer needed with this full integration." }, { "commit": "1e83f2a022b36191c2d12185ed57cf4d96462862", "date": "2022-07-28 14:53:48 -0400", "subject": "Add coverage to C test harness.", "body": "Coverage reports are still generated in Perl, but all the settings have been added to the C harness to generate raw coverage data." }, { "commit": "c99ea54f17c1f26a5e996bb8cfd3df81684f813b", "date": "2022-07-27 10:32:32 -0400", "subject": "Integrate C test harness with Perl test harness.", "body": "The C test harness is used for unit tests from the Perl harness where possible. Currently, unit tests can be run in the C harness when --no-coverage is specified and --profile is not specified.\n\nC harness tests work on meson 0.45.\n\nThe C harness runs with valgrind by default. Valgrind can be disabled with --no-valgrind.\n\nAlso rebuild containers to add meson and update the documentation so that meson builds will work (even though we don't do them yet)." }, { "commit": "2caef37fd55ea12959015f02a106774f6ad711b7", "date": "2022-07-26 18:25:24 -0400", "subject": "Add reviewer for c267ba51." }, { "commit": "e9ff5248039eedb25da2fee550ec64fcebbc51c7", "date": "2022-07-26 15:15:12 -0400", "subject": "Add absolute path for kill in pgbackrest.service." }, { "commit": "79d9884141f90a9e93fb2ec8faef15e460dd3442", "date": "2022-07-25 09:39:54 -0400", "subject": "Move release build check to src/meson.build.", "body": "This allows the C harness to perform release builds which are required for some tests." }, { "commit": "c267ba51b1778b5872f4ee928a9614545d743f3f", "date": "2022-07-22 16:24:55 -0400", "subject": "Move standby timeline check after checkpoint.", "body": "The standby timeline check was being performed using pg_control data loaded before the backup started. If the backup was started immediately after a promotion the standby might not have executed a checkpoint and written the new timeline to pg_control.\r\n\r\nInstead perform the timeline check after the checkpoint is executed. This should ensure that the new timeline is in pg_control." }, { "commit": "cbbe93f592d8097393c5ffe214f0a8c5fb0549c1", "date": "2022-07-22 14:45:38 -0400", "subject": "Improve warning message on backup resume.", "body": "The prior warning made it sound as if some action was required on the part of the user.\n\nThe new message should make it clearer that this action will be performed by pgBackRest." }, { "commit": "4c47cc5d4786b88a06c38aefe628356342433f64", "date": "2022-07-22 09:24:29 -0400", "subject": "Remove Debian package patch now that it has been merged upstream." }, { "commit": "19d9941367f68d6effb516fbb1444877fea9a9ce", "date": "2022-07-21 20:10:51 -0400", "subject": "More C test harness improvements and CI.", "body": "Build pgbackrest binary and auto-generated code automatically.\n\nRemove --module option and allow modules to run by parameter. This is less verbose and multiple modules can be run at a time.\n\nAllow filtering of modules. Multiple tests can be passed as parameters and if the module ends in / it will be used as a prefix filter. For example, common/ will run all the common modules.\n\nIf a test errors the remaining tests will still run but the test process will eventually exit with an error.\n\nCI tests are included but unit tests remain on the development branch.\n\nWith these changes all unit tests run except those that specify the define (e.g. common/assert-off) or containerReq (e.g. protocol/protocol) keywords.\n\nBuilding the C test harness has been simplified:\n\nmeson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug test/build/none pgbackrest\nninja -C test/build/none test/src/test-pgbackrest\n\nTo run all modules:\n\ntest/build/none/test/src/test-pgbackrest test\n\nJust the common/error module:\n\ntest/build/none/test/src/test-pgbackrest test common/error\n\nAll info modules:\n\ntest/build/none/test/src/test-pgbackrest test info/" }, { "commit": "edfcf1652c6ab07fe5a147b97df0c563e09d3445", "date": "2022-07-21 19:27:58 -0400", "subject": "Test Dockerfile improvements.", "body": "Add tzdata package so timezone tests in command/restore work correctly.\n\nMark default git path as safe. This is a security fix that is not applicable in this environment, but must be set.\n\nAlso remove package cleanup, which is inconvenient when new packages need to be installed. It makes sense for containers that will be downloaded from Dockerhub but not so much for a locally-maintained container." }, { "commit": "5e5b04be3741b4e903c9a7764dadfbe5bda121ae", "date": "2022-07-21 18:19:47 -0400", "subject": "Fix common/lock test creating \"750\" path.", "body": "This was clearly an attempt to set the mode when creating a directory, but it never worked and instead created a \"750\" directory in the current working directory.\n\nDetected when running in an environment where the current working directory was read-only." }, { "commit": "f9bbafbf3fbd67fc289cf849900799b4718dace1", "date": "2022-07-20 19:02:14 -0400", "subject": "C test harness improvements.", "body": "Add harness depends when present.\n\nInclude libyaml in all test builds.\n\nFix mode on paths before trying to remove and set test path with mode 770 to match the Perl test harness.\n\nWith these changes all unit tests run except those that specify the define (e.g. common/assert-off), binReq (e.g. command/archive-get), or containerReq (e.g. protocol/protocol) keywords.\n\nBuilds and code generation need to be done in advance. The following commands are required for setup:\n\nmeson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug build pgbackrest\nninja -C build test/src/test-pgbackrest\nbuild/src/build-code help pgbackrest\nbuild/src/build-code postgres pgbackrest\n\nNow tests can be run, e.g.:\n\nbuild/test/src/test-pgbackrest --module=postgres/interface" }, { "commit": "c625f05a13e0238bf06feb26d0d8757dd2bedffd", "date": "2022-07-20 17:45:39 -0400", "subject": "Unify code builder binaries into a single binary.", "body": "Creating new binaries was convenient at first but has now become a maintenance issue.\n\nSolve this by combining that into a single binary that takes an additional parameter to indicate which code should be built.\n\nAlso clean up path handling to make it easier to build code from the command line." }, { "commit": "7eb5d679857999faebe78730c5a30e450ef80d5c", "date": "2022-07-19 18:03:39 -0400", "subject": "Add module harness and shim support to the C test harness.", "body": "This allows running unit tests up to common/ini, excluding common/debug-off and common/assert-off." }, { "commit": "8fdeed780787a2c6df6672f5d9c75fb7b5b613bc", "date": "2022-07-19 17:57:13 -0400", "subject": "Copy repository links as files for testing.", "body": "This makes the test code a bit simpler where we are listing a path but not following links.\n\nLinks in the repository can be used for testing but should never be committed to the main branch." }, { "commit": "eb775790087cd8594f6649893895c42a407302cd", "date": "2022-07-19 17:25:07 -0400", "subject": "Fix comment typo." }, { "commit": "f3a10b921bf3d87152d27347df23b487877abe94", "date": "2022-07-18 10:52:57 -0400", "subject": "Update config.guess to latest version." }, { "commit": "20f9f726478a4d1c8897cc31a5bf7df52275b84f", "date": "2022-07-18 10:49:58 -0400", "subject": "Begin v2.41 development." }, { "commit": "1ff531090b9d1eccd9107592398148a51486c9f8", "date": "2022-07-18 09:32:30 -0400", "subject": "v2.40: OpenSSL 3 Support" }, { "commit": "1df1e0281b94a06ff0f16622e2cbb76598d430d1", "date": "2022-07-14 17:13:33 -0400", "subject": "Replace AC_PROG_CC_C99 configure macro with AC_PROG_CC.", "body": "AC_PROG_CC_C99 is now obsolete.\n\nThe AC_PROG_CC macro does not guarantee C99 compliance but we add an option to check for that a bit later." }, { "commit": "364af1635d40431276cc6fd87d6ddd689b611e09", "date": "2022-07-14 08:26:03 -0400", "subject": "Force target-timeline=current when restore type=immediate.", "body": "Explicitly set target timeline to \"current\" when type=immediate and PostgreSQL >= 12. We do this because type=immediate means there won't be any actual attempt to change timelines, but if we leave the target timeline as the default of \"latest\" then PostgreSQL might fail to restore because it can't reach the \"latest\" timeline in the repository from this backup.\r\n\r\nThis is really a PostgreSQL bug and will hopefully be addressed there, but we'll handle it here for older versions, at least until they aren't really seen in the wild any longer.\r\n\r\nPostgreSQL < 12 defaults to \"current\" (but does not accept \"current\" as a parameter) so no need set it explicitly." }, { "commit": "75623d4583bfdaccb3a19b935fd04d744b363923", "date": "2022-07-08 17:21:39 -0400", "subject": "Create snapshot when listing contents of a path.", "body": "Previously a callback was used to list path contents and if no sort was specified then a snapshot was not required. When deleting files from the path some filesystems could omit files that still existed, which meant the path could not be removed.\r\n\r\nFilter . out of lists in the Posix driver since this special entry was only used by test code (and filtered everywhere in the core code).\r\n\r\nAlso remove callbacks from the storage interface and replace with an iterator that should be easier to use and guarantees efficient use of the snapshots." }, { "commit": "f9ac53db92574e8b499187ee4f2a7258bc2f4794", "date": "2022-07-08 14:38:14 -0400", "subject": "Fix typo." }, { "commit": "74a4ac801db921f7b11b63b9f927228c6a6af861", "date": "2022-07-08 11:13:55 -0400", "subject": "Add link to PostgreSQL configuration in repository host section.", "body": "This should make the documentation clearer when starting from this section." }, { "commit": "0eccbc8bf47073ab8ec1f13fed9865d0bd7c3635", "date": "2022-07-06 18:17:52 -0400", "subject": "Meson builds work on version 0.45.", "body": "v0.45 ships with Ubuntu 18.04, which is currently the oldest distro we support. We may never do a Meson release on Ubuntu 18.04 but this allows us to start running unit tests with Meson in the meantime.\n\nSome more granular options are not available so we use buildtype in more places.\n\nThe check for a in-tree autoconf/make build had to be removed since the filesystem APIs are not available.\n\nFinally, alias_target was removed. This means that full paths must be used for build targets, which does not seem too bad. For instance, test/src/test-pgbackrest must now be used as a build target instead of simple test-pgbackrest." }, { "commit": "72960bbf179b4e8051bd4fc50be57d95159e9e13", "date": "2022-07-06 09:55:32 -0400", "subject": "Rename strTrunc() to strTruncIdx() and add strTrunc().", "body": "strTrunc() is now shorthand for truncating to index 0. This is convenient when a string is being reused." }, { "commit": "cd25cec1471dde60f85404744252b675efff2522", "date": "2022-07-05 14:34:31 -0400", "subject": "Add comment to indicate that type is in a different info level." }, { "commit": "1c0bf0b15da8c0d15cc639851ff1d1bd05b51d06", "date": "2022-07-05 14:28:40 -0400", "subject": "Reorder null user/group checks in remote protocol and add tests.", "body": "Coverage for these checks was dependent on the order the files were read from disk, which made the tests fragile.\n\nRearrange the checks and add a test that won't depend on order." }, { "commit": "326d152a14121cb298089847a3694aa9f50bcbd2", "date": "2022-07-05 06:58:19 -0400", "subject": "Update contributor name." }, { "commit": "845d82f6825d4101e6a465a435ef056ff6ec15a2", "date": "2022-07-01 20:32:10 -0400", "subject": "Use S3 IsTruncated flag to determine list continuation.", "body": "Previously we were just checking for the existence of NextContinuationToken, which the S3 documentation indicates will not be present when the list is not truncated. However, recent versions of Scality send a blank NextContinuationToken when IsTruncated is false. Sending the blank continuation token back causes Scality to send another blank continuation token and an infinite loop occurs.\r\n\r\nInstead use IsTruncated (which is required to be present) to determine whether NextContinuationToken should be present. Error if NextContinuationToken is then missing or empty, since an empty token caused an infinite loop with the Scality server (which arguably should have errored when passed an empty token)." }, { "commit": "61ca9b58a000aa8789ef3267682519bdb15e8a6b", "date": "2022-06-28 14:15:08 -0400", "subject": "Replace hrnStorageInfoListCallback() with TEST_STORAGE_LIST() in tests.", "body": "The TEST_STORAGE_LIST() macro is more robust and hides the callback mechanism from the caller.\n\nAdd features to TEST_STORAGE_LIST() that hrnStorageInfoListCallback() had.\n\nUpdate tests to use the abbreviated type output (e.g. path/) generated by TEST_STORAGE_LIST()." }, { "commit": "59f148bf6ea8c090d4d65ce5b135a71501e8ae22", "date": "2022-06-26 08:42:43 -0400", "subject": "Generate -fmacro-prefix-map to strip relative path from __FILE__.", "body": "This provides reproducible builds and minimizes the file path in debug messages, just like an in-tree make build.\n\nFor test source, prefix with test/ in case there are any module name collisions." }, { "commit": "f863fc98886f4d6c2803b9a2396539597d0d7be2", "date": "2022-06-23 12:20:56 -0400", "subject": "Add experimental unit test harness written in C.", "body": "Having the test harness in C will allow us to remove duplicated Perl code and test on systems where Perl support is not present.\n\nCustom harnesses and shims are currently not implemented, which means only the following tests in the common module will run: error, stack-trace, type-convert, assert-on, mem-context, time, encode, type-object, type-string, type-list, type-buffer, type-variant, reg-exp, log.\n\nThe experimental test harness is being committed with partial functionality so it can be used in Windows development. The remaining features will follow as needed." }, { "commit": "b7a1b3ec2c16fde6a201f404a27d8343595900b6", "date": "2022-06-23 10:57:58 -0400", "subject": "Used DEBUG instead of NDEBUG in common/stackTrace module.", "body": "In some testing cases these might not be in sync, which causes unpredictable behavior." }, { "commit": "29d2f0f9fda4dbba0dc5eca6f84368a6f655bd28", "date": "2022-06-22 09:52:30 -0400", "subject": "Add cast to handle compilers (e.g. MSVC) that coerce to signed int.", "body": "MSVC changes the sign in this case, presumable because of the subtraction. Cast so that MSVC does not also trigger a mixed sign warning." }, { "commit": "9fd85a104ad56061b028afd9057ce8c1d636d3dd", "date": "2022-06-21 09:50:38 -0400", "subject": "Disable meson for all but debug builds.", "body": "The meson builds are still experimental so for now the configure/make build process is preferred for release builds. This message should help prevent any automated build systems from picking up meson instead." }, { "commit": "665da12ae7f519077512e60ce3a5ec6c6a9997fd", "date": "2022-06-17 16:41:48 -0400", "subject": "Update meson.build include comment to be more general.", "body": "Eventually multiple subdirs will be included so update the comment to reflect that." }, { "commit": "5ecae90f0254ef752d8323ea5be3b99673ec46f7", "date": "2022-06-17 16:31:48 -0400", "subject": "Use constants rather than replacements when possible in test.c.", "body": "Some of the replacements that were being done already existed as constants, so use the constants instead.\n\nAlso fix a minor formatting error introduced when testAdd() was renamed to hrnAdd()." }, { "commit": "dab1e4b6c66018c03c6e01309c41a89ef82ff5bc", "date": "2022-06-17 16:27:31 -0400", "subject": "Add strReplace().", "body": "Allows substrings to be replaced with another string." }, { "commit": "fb9acc1c93d504d366493bc42038f15faa80300d", "date": "2022-06-17 16:17:02 -0400", "subject": "Add higher level YAML functions.", "body": "These functions make parsing YAML simpler." }, { "commit": "55bcb933eee753a0cdd13f9861a00c33d194c1e7", "date": "2022-06-17 11:17:52 -0400", "subject": "Move protocol module from common to command.", "body": "This module has dependencies on command/command so it does not make sense for it to be in the common module. Also move protocolFree() to main() since this is a very large dependency.\n\nAdjust the tests so command/exit can be tested later. This is a bit messy but will get adjusted as we improve the test harness." }, { "commit": "eb72a80b471145d97ac4170e3215f7c67aa83082", "date": "2022-06-15 09:06:25 -0400", "subject": "Fix continuation character spacing." }, { "commit": "716bba5800d504a76d4bdc99588448824be3d83f", "date": "2022-06-14 08:13:22 -0400", "subject": "Fix hard-coded WAL size assertion in walSegmentNext().", "body": "PG_WAL_SEGMENT_SIZE_DEFAULT is used to compare and check WAL size on pre-11 installations. However, there is a hard-coded assertion in walSegmentNext() which doesn't respect PG_WAL_SEGMENT_SIZE_DEFAULT.\r\n\r\nUpdate the assertion to use PG_WAL_SEGMENT_SIZE_DEFAULT." }, { "commit": "282edda661b5f1eb189998e7db680b91a2b8a66b", "date": "2022-06-09 17:42:00 -0400", "subject": "Remove storageLocal() dependency from common/lock module.", "body": "The storage/helper module is a very heavy dependency to introduce in the common module. Creating Posix storage objects is cheap so just do that instead." }, { "commit": "04f5ef25faf276e51bd546199f1197b5c70003e8", "date": "2022-06-09 17:18:32 -0400", "subject": "Add valgrind to Dockerfile missed in a16cf5ea." }, { "commit": "79443bea3634f7dd3858b6f65838e31fb9b87c3f", "date": "2022-06-09 16:55:07 -0400", "subject": "Move bldEnum() to the build/common/render module.", "body": "This function will be useful for other build/test modules so extern it.\n\nAlso skip the first upper-case when no prefix is provided." }, { "commit": "1a00ab10033e9ac6e6f797627567e8b91e143432", "date": "2022-06-09 07:34:11 -0400", "subject": "Fix compile error when DEBUG_EXEC_TIME is defined without DEBUG.", "body": "If DEBUG is not defined then the ASSERT() macro expands to nothing. In this case the timeBegin variable is never used and a compilation error occurs.\r\n\r\nThis test should work without DEBUG defined so use CHECK() instead of ASSERT()." }, { "commit": "0dabf88e9d85e4db2ff65ebbb14b4320e1092a8b", "date": "2022-06-08 17:43:23 -0400", "subject": "Add FN_NO_RETURN macro.", "body": "Change all instances of __attribute__((__noreturn__)) to a macro in meson.build / build.auto.h.in.\r\n\r\nAs compiler attributes written in the form of __attribute__ are not supported by MSVC, this is one of several commits to make the code-base more robust and allow using MSVC-style attributes later." }, { "commit": "8babd558bc3a5b930fd096ffb306551773f1f14d", "date": "2022-06-08 07:26:15 -0400", "subject": "Add missing build.auto.h includes.", "body": "These are required for proper building but were probably not noticed before since they are not part of the core code." }, { "commit": "a16cf5eac7217e79d8669fab5dff1863e26208c9", "date": "2022-06-06 16:32:20 -0400", "subject": "Update CI to use Ubuntu 22.04 and Fedora 36.", "body": "Both have newer gcc and OpenSSL 3.\n\nFedora 36 runs horribly slow with valgrind enabled so run the valgrind tests on Ubuntu 22.04. Fedora 36 has a newer gcc so it is still worth testing on." }, { "commit": "08242ee6ac93b9eb9c05cc39e3db15bbafb9052c", "date": "2022-06-06 14:47:47 -0400", "subject": "OpenSSL 3 support.", "body": "There are two changes:\r\n\r\n* Suppress deprecation warnings so we can build with -Werror and -Wfatal-errors. At some point we'll need to migrate to the new APIs but there does not seem to be a good reason to support two sets of code right now.\r\n\r\n* Update the handling for unexpected EOF to handle EOF or error. The error code for EOF has changed and become harder to identify, but we probably don't care whether it is an error or EOF." }, { "commit": "f92ce674f79e2d6056794c1af4002c01dad93aed", "date": "2022-06-06 13:52:56 -0400", "subject": "Automatically create PostgreSQL version interfaces.", "body": "Maintaining the version interfaces was complicated by the fact that each interface needed to be in separate compilation unit to avoid type conflicts. This also meant that various build/test files needed to be updated to add the new interfaces.\r\n\r\nSolve these problems by auto-generating all the interfaces into a single file. This is made possible by parsing defines and types out of the header files and creating macros to rename the types. At the end of the version interface everything is undef'd. Another benefit is that the auto-generated interfaces can be static and included directly into postgres/interface.c.\r\n\r\nSince some code generation is now always required for tests, change --no-gen to --min-gen in test.pl.\r\n\r\nIt would also make sense to auto-generate the version defines in postgres/version.h, but that will be left for a future commit." }, { "commit": "b8fc20d5b8e7ca003ff39f29ea6e3ba57653f55c", "date": "2022-06-03 14:13:56 -0400", "subject": "Add experimental Meson build.", "body": "Meson is a new build system that offers simpler syntax and superior performance to autoconf/make. In addition, Windows is supported natively.\r\n\r\nThe Meson build appears complete, but currently is used only for auto-generation of code and the host build of pgbackrest. Some container upgrades will be required before Meson can be used for container builds.\r\n\r\nAlso patch the Debian package to force autoconf/make rather than Meson." }, { "commit": "148956aed8aca201aa800af321d946094b96f176", "date": "2022-06-01 10:13:57 -0400", "subject": "Remove useless command/check unit test.", "body": "This test was a placeholder and did not provide any coverage, but it did give inconsistent results on different shell versions." }, { "commit": "29b2a54fcc74457af5c63ffb505c5fbb24bb2ade", "date": "2022-05-31 17:28:58 -0400", "subject": "Allow any path to be passed to the --test-path option.", "body": "A hard-coded path prevented this from working correctly." }, { "commit": "be354c489017ea84ed825f226b50d170bec1ff68", "date": "2022-05-31 16:52:26 -0400", "subject": "Update CodeQL to version 2.", "body": "Version 1 will be deprecated on December 12, so upgrade now to avoid nasty surprises." }, { "commit": "2c38c9a56f95efea42d489792751b77acfc470e4", "date": "2022-05-31 16:43:18 -0400", "subject": "Skip stopping PostgreSQL 9.1 in real/all integration test.", "body": "Stopping the cluster has started consistently running out of memory on PostgreSQL 9.1. This seems to have happened after pulling in new packages at some point so it might be build related.\n\nStopping the cluster is not critical for 9.1 so skip it." }, { "commit": "2feaaeaac803d6c0193477b56dbeb622fd991156", "date": "2022-05-31 16:06:41 -0400", "subject": "Add .inc extension to C files included in other C files.", "body": "These files were never intended to be compiled on their own so the .c extension was a bit misleading. In particular Meson does not like .c files that are not intended to be compiled independently.\n\nLeave header files as is since they are already protected against being included more than once and are never expected to be compiled." }, { "commit": "cb891fa2d402ee338b72d2c0962cc5cd49eea4f1", "date": "2022-05-31 13:02:51 -0400", "subject": "Run remaining Github Actions CI on Ubuntu 20.04.", "body": "The s390x/ppc64le tests are already running on 20.04." }, { "commit": "2643050be0dc4b7363c434f58373919bd6fdac68", "date": "2022-05-31 12:36:21 -0400", "subject": "Skip internal options in the configuration reference.", "body": "Most internal options were being skipped, but not in the case where an option was marked internal for a specific command.\n\nThe command-line help was not affected by this issue." }, { "commit": "f0aae6ceab9a95c6d63b8955abd2592ddbbce90a", "date": "2022-05-31 12:26:57 -0400", "subject": "Remote repo options should not be valid for the backup command.", "body": "Some of the remote repo options were gated by repo-local, but the rest relied on repo-host-cmd.\n\nRemove backup from the repo-host-cmd option since none of the dependent options are valid for backup." }, { "commit": "a902c7808d8bf651389e2479d4622b0fd49ab09d", "date": "2022-05-31 12:24:21 -0400", "subject": "Make all repo-* options visible for stanza create/update/delete.", "body": "31c7824a allowed these commands to run remotely but neglected to remove some internal flags, which prevented all the repo-* options from being visible in the documentation." }, { "commit": "60d70fa66fc3f709914f40dd088bac1b467001f2", "date": "2022-05-25 18:18:37 -0400", "subject": "Disable incremental manifest save when resume=n.", "body": "The manifest is saved on a regular basis during a backup so a failed backup can be resumed. For backups that the user has configured/invoked as not resumable, skip the incremental save of the manifest." }, { "commit": "8c2b3a044fa175031366fb4f676b5c1a0a319ec0", "date": "2022-05-25 15:27:53 -0400", "subject": "Use StringId for type in cryptoHash() and cryptoHmacOne().", "body": "This brings hash types in line with cipher types (i.e. a StringId enum) and allows removal of some Strings." }, { "commit": "38ad2838d553febae1b7fe0244d942dfaa9cf548", "date": "2022-05-25 14:20:19 -0400", "subject": "Set backup percent complete to zero before copy start.", "body": "Waiting to write percent complete until the first file completed resulted in a period of time where the backup was running without status available to the user.\r\n\r\nRemedy this by initializing percent complete to zero when the backup is ready to start copying files." }, { "commit": "7b1935c71005a5a928e34caf7b8b48e5d871cc66", "date": "2022-05-25 10:23:43 -0400", "subject": "Remove unused errors.", "body": "Most of these were probably never ported from Perl to C and others became obsolete over time.\n\nFix one error that was the wrong type.\n\nAlso fix/improve some comments." }, { "commit": "f6f2f2e2a34c01e20bbf6c26e5e8a232a6bd232d", "date": "2022-05-25 09:42:49 -0400", "subject": "Add missing static keywords.", "body": "Add static keyword to local variables where missing." }, { "commit": "7ec51e7e6279ea576ad3b4b5c2ea1fc389d08ab5", "date": "2022-05-24 16:39:35 -0400", "subject": "Truncate files during delta restore when they are larger than expected.", "body": "Previously the behavior was to download the file from the repository when it was not exactly the same size in PGDATA. However, it may just be that the file was extended and the contents are the same up to the file size recorded in the manifest. This could also be very valuable for files that are always append only, like logs.\r\n\r\nChange info.size to file->size in one place. Both are technically correct but file->size makes more sense.\r\n\r\nUse the new fileName variable in a few existing places.\r\n\r\nAlso adjust some existing comments to make them clearer." }, { "commit": "c98baab6b56967e222b825e22e5808e17c02d025", "date": "2022-05-19 18:24:09 -0400", "subject": "New CI container builds for PostgreSQL 15 beta1 and minor releases.", "body": "Remove VM_OS_REPO since it is no longer required.\n\nRebalance PostgreSQL versions for more efficient test times.\n\nAlways print version of PostgreSQL when testing. This helps verify that new minor releases are being used." }, { "commit": "69adb990dced3bf1d76a3e2482aad38016165d6c", "date": "2022-05-19 12:25:58 -0400", "subject": "Use storagePathP() instead of cfgOptionStr() to get base repo path.", "body": "cfgOptionStr() may not have the correct value if the repo is remote.\n\nUse storagePathP() instead since it can ask the remote for the correct value when required." }, { "commit": "c7a66ac1afb9a55b049dc18493b602b7bf62bda4", "date": "2022-05-18 10:52:01 -0400", "subject": "Improve memory usage of mem contexts.", "body": "Each mem context can track child contexts, allocations, and a callback. Before this change memory was allocated for tracking all three even if they were not used for a particular context. This made mem contexts unsuitable for String and Variant objects since they are plentiful and need to be as small as possible.\r\n\r\nThis change allows mem contexts to be configured to track any combination of child contexts, allocations, and a callback. In addition, the mem context can be configured to track a single child context and/or allocation, which saves memory and is a common use case.\r\n\r\nAnother benefit is that Variants can own objects (e.g. KeyValue) that they encapsulate. All of this makes memory accounting simpler because mem contexts have names while allocations do not. No more memory is used than before since Variants and Strings still had to store the memory context they were originally allocated in so they could be easily freed.\r\n\r\nUpdate the String and Variant objects to use this new functionality. The custom strFree() and varFree() functions are no longer required and can now be a wrapper around objFree().\r\n\r\nLastly, this will allow strMove() and varMove() to be implemented and used in cases where strDup() and varDup() are being used to move a String or Variant to a new context. Since this will be a bit noisy it is saved for a future commit." }, { "commit": "83af3f1b7ae78b150849939766e2e555366b95d5", "date": "2022-05-18 08:48:48 -0400", "subject": "Add additional detail to warnings when delta checksum is auto-enabled.", "body": "Hopefully this will help with debugging when it is not clear why delta checksum is being enabled." }, { "commit": "5dfd00bb6ce0d6d2ead0ba6c899c18983f9a7ac9", "date": "2022-05-18 08:18:34 -0400", "subject": "Fix RHEL container build for documentation.", "body": "For some reason /lib/systemd/system/sysinit.target.wants no longer exists in the rockylinux:8 container.\n\nCreate this directory explicitly in case it does not exist." }, { "commit": "5360f2ec0af08ac1755d48f234347c240ff09f4a", "date": "2022-05-16 10:50:07 -0400", "subject": "Fix comment indentation." }, { "commit": "b598f49dedbfb2bdd19d43dae9fa9ba654f2287c", "date": "2022-05-16 09:17:00 -0400", "subject": "Udpate parse.auto.c with labels from 4dcc9df2.", "body": "Committed separately so it can be ignored in history/blame." }, { "commit": "4dcc9df222d2ecef6721c9ecd52b05e24024cc3d", "date": "2022-05-16 09:14:46 -0400", "subject": "Add labels in parse.auto.c to make diffs easier to read.", "body": "Because there is a lot of repetition in this file, changes can look very jumbled with existing data in a diff. Also, if can be hard to tell what is being modified if the diff does not show enough lines before and after.\n\nThis change adds labels to the end of the line to localize the diff and make it easier to see what has been changed. Also, remove some linefeeds and make separators more consistent.\n\nThe change to parse.auto.c will be committed separately so it can be ignored in history/blame." }, { "commit": "f5023a769d4580258a336cf26f2b293aa2341b29", "date": "2022-05-16 08:54:19 -0400", "subject": "Update config.guess to latest version." }, { "commit": "243eef1e520bb72ae2b28a627dda323305e38862", "date": "2022-05-16 08:51:37 -0400", "subject": "Begin v2.40 development." }, { "commit": "901e829f6dd05fdc15d899bc7f1108883002b1d5", "date": "2022-05-16 08:46:24 -0400", "subject": "v2.39: Verify and File Bundling" }, { "commit": "19dd015d589ed3efd31a2a238249a150fb3748c1", "date": "2022-05-13 09:41:53 -0400", "subject": "Fix issues in improved path validation for repo-* commands.", "body": "If the user requested the exact repo path then strSub() would be passed an invalid start value leading to an assertion:\r\n\r\n$ pgbackrest --stanza=test repo-ls /var/lib/pgbackrest\r\nASSERT: [025]: start <= this->pub.size (on dev builds)\r\nASSERT: [025]: string size must be <= 1073741824 bytes (on prod builds)\r\n\r\nFix this by checking if the requested path exactly equals the repo path and returning an empty relative path in this case.\r\n\r\nAnother issue was that invalid subpaths were not detected if they started with the repo path. For example, /var/lib/pgbackrestsub would not generate an error if the repo path was /var/lib/pgbackrest. Fix this by explictly checking for a / between the repo path and the subpath. This also requires special handling when the repo path is /.\r\n\r\nThis is not a live bug since the issues were found in an unreleased feature introduced in 5ae84d5." }, { "commit": "024500782e9fde91cf236db70a584beb52586d38", "date": "2022-05-12 17:02:08 -0400", "subject": "Reduce encrypted WAL segments sizes in command tests.", "body": "The encrypted archive-push and repo tests were running very slowly on 32-bit with Valgrind enabled. This appears to be an issue with a newer version of Valgrind, but it has been going on long enough that bisecting does not seem to be worthwhile.\n\nReduce the size of the encrypted test segments where possible to improve overall test performance." }, { "commit": "7d9b2e267c3f68463000cfcfec6018cc12203f23", "date": "2022-05-12 11:57:12 -0400", "subject": "Move 32-bit CI testing from Debian 9 to Ubuntu 18.04.", "body": "Debian 9 will be EOL before our next release, so move 32-bit testing to Ubuntu 18.04, which is older than Debian 10." }, { "commit": "24f138b0672ed55aeef1673b816b803487bca932", "date": "2022-05-11 16:20:41 -0400", "subject": "Remove excessive const usage in build module.", "body": "This was an experiment that attempted to create immutable structs (at least without casting). It turned out to be a bit burdensome and required unsafe-looking casting in some cases." }, { "commit": "3d8ee552fe85aae6f2f00cd2a9436fb6b34513e1", "date": "2022-05-11 15:59:37 -0400", "subject": "Move some PostgreSQL integration tests to rh7.", "body": "This makes the u20 test run faster. Since u20 is currently the longest running test, the time to complete all tests is also improved." }, { "commit": "a913113fda5886ffd2ff136cdd52a3f469eca026", "date": "2022-05-11 15:23:41 -0400", "subject": "Add option type descriptions.", "body": "This cuts down on repetition of the size descriptions and adds basic descriptions for the other option types." }, { "commit": "5fbea6da811ee93dcd966b8f7d84d6da2e65c6c5", "date": "2022-05-11 10:39:31 -0400", "subject": "Add br tag for documentation.", "body": "This tag allows for a simple linefeed in a p tag instead of being forced to start a new paragraph." }, { "commit": "8ee85bc605da4aff5069bd1b965bbb585d9fe851", "date": "2022-05-11 08:46:23 -0400", "subject": "Fix \"that that\" typos." }, { "commit": "c4f7edef2b6a9b541e94793c1dabbff7db854a9b", "date": "2022-05-11 08:42:46 -0400", "subject": "Fix typos in help." }, { "commit": "50d409a812be577f2c296f3d6096cf97fab049ff", "date": "2022-05-10 14:17:05 -0400", "subject": "Add FAQ about backup types and restore speed.", "body": "Based on several questions/misunderstandings, provide clarification about the backup type only affecting the backup action, and not the restore." }, { "commit": "de816a0f5793747a774b98af627776112cc7cb2d", "date": "2022-05-10 13:18:26 -0400", "subject": "Remove integration expect log testing.", "body": "Integration expect log testing was originally used as a rough-and-ready way to make sure that certain code paths were being executed before the unit tests existed. Now that we have 100% unit test coverage (with expect log testing) the value of the integration expect tests seems minimal at best.\r\n\r\nBut they do cause numerous issues:\r\n\r\n- Maintenance of the expect code and replacements that are required to keep logs reproducible.\r\n- Even a trivial change can cause massive churn in the expect logs, e.g. d9088b2. These changes should be minutely audited but since the expect logs have little value now it is seldom worth the effort.\r\n- The OS version used to do expect testing (RHEL7) can only be used to test one version of PostgreSQL. This makes it hard to balance the PostgreSQL version testing between OS versions.\r\n- When a commit affects expect logs it is not clear (especially for new developers) how to regenerate them and our contributing guide is silent on the issue.\r\n\r\nThe goal is to migrate the integration tests to C and expect testing is not part of that plan. It seems best to get rid of them now." }, { "commit": "3a403944729e03c66de83e6750c38cab97743333", "date": "2022-05-10 11:18:12 -0400", "subject": "Remove obsolete test in common/memContext.", "body": "Once upon a time the allocation array was allocated up front so this test was required for the top context, which did not allocate up front.\n\nNow allocations are done on demand so this case is covered for every context that does not allocate memory." }, { "commit": "6b98b3534e15a8d277547eb2932c085bc7515eec", "date": "2022-05-10 06:52:56 -0400", "subject": "Fix typo." }, { "commit": "cc5b0614894e1b5ef78f1ceba3bc8e4760a40a24", "date": "2022-05-09 18:07:11 -0400", "subject": "Document required base branch for pull requests.", "body": "Be explicit when submitting a PR about which branch to use as the base." }, { "commit": "b4c1ca7b807a906d42d540307da3603052c4a674", "date": "2022-05-09 14:19:05 -0400", "subject": "Split 32-bit CI tests.", "body": "This helps rebalance some of the tests that are running long, i.e. d9 and u20.\n\nI would be better to move more PostgreSQL versions to d9, but the base VM does not contain more versions. New minor versions will be out later in the week so that seems a better time to be rebuilding containers." }, { "commit": "39dddbb6bc7d296fe25a7210468b2f51ea6b1816", "date": "2022-05-09 12:48:19 -0400", "subject": "Add limited CI for ppc64le/s390x using emulation.", "body": "The emulation is so slow that running all the unit tests would be too expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to test on one big-endian architecture to be sure that checksums are correct.\n\nUpdate checksums in the tests where they had gotten out of date since the last time we were testing on s390x. Also use a different test in command/archivePushTest to show the name of the file when a checksum does not match to aid in debugging.\n\nThe command/archive-push test was updated but not included because there is also a permissions issue, which looks to be the same as what we see on MacOS/FreedBSD. Hopefully we'll be able to fix all of those at the same time." }, { "commit": "eefa0b161a6a02b7843e18455443a4bce41cd4b9", "date": "2022-05-09 11:59:08 -0400", "subject": "Simplify messaging around supported versions in the documentation.", "body": "The version ranges given in the user guides caused confusion. For example, because the user guide for RHEL specified PostgreSQL 9.6-11, users questioned whether pgBackRest worked for PostgreSQL 12 on RHEL.\r\n\r\nRemove these ranges and add more explanatory text to the introduction to try and make it clearer how the user guides work and which versions are covered (basically all of them)." }, { "commit": "ef4c4ab8525f81c9ed22c7e82ed12096e81703c5", "date": "2022-05-09 10:39:43 -0400", "subject": "Use variable instead of function to track FINALLY() state.", "body": "The function worked fine, but Coverity was unable to determine that the finally block was run, which led to false positives about unfreed memory.\r\n\r\nUsing a boolean in the block makes it clear to Coverity that the finally block will always be run no matter what else happens.\r\n\r\nWe'll depend on the compiler to optimize away the boolean if it is not used in a finally block. The cost of the boolean is fairly low in comparison to everything else being done in these macros, so it does not seem worth having a separate block even if the compiler is not able to eliminate the boolean.\r\n\r\nThis reverts most of 9a271e9 that fixed a bug caused by c5b5b58, which was also attempting to help Coverity understand FINALLY() blocks." }, { "commit": "e8c40a24df14fcf281de672c19d0a55dc7cb50f6", "date": "2022-05-09 09:56:19 -0400", "subject": "Remove unnecessary TRY() block in common/regExp module.", "body": "This code was written before MEM_CONTEXT_TEMP*() was available, which is a better solution." }, { "commit": "4d8c36715ddd67c11d1ab166dcd0b4850631d83d", "date": "2022-05-06 19:44:46 -0400", "subject": "Remove legacy Travis-CI configuration.", "body": "Travis-CI is now strictly a paid service. Multiple attempts to use their \"free\" service have failed due to lack of community credit and general issues with their plugin.\n\nRemove the configuration so it does not appear we are testing on Travis-CI." }, { "commit": "46b7b7287446d0bb5ed0963fd01c51fb2b08893f", "date": "2022-05-06 18:23:36 -0400", "subject": "Add hint when unable to find the primary cluster.", "body": "If all available clusters are in recovery, pgBackRest will not be able to find a primary for the backup." }, { "commit": "53bfdbc01ece396c802022fea2216d3b9d7beaa1", "date": "2022-05-06 16:02:44 -0400", "subject": "Remove useless test in config/parse unit test.", "body": "Since the packSize field is 7 bits, it could never fail the check for > 127.\n\nThe compiler will catch any packs that are larger than 7 bits and then the pack size will need to be adjusted. For now just adjust the comment to reflect what the test does and give a clearer indication of what to do when a pack grows too large." }, { "commit": "77311a9af7870e79d7a3c9ecadc268c2364458c8", "date": "2022-05-06 15:38:03 -0400", "subject": "Fix indentation.", "body": "gcc11 complains about this indentation being misleading." }, { "commit": "efbcd975c4c5b4c29c937154e2b1e221aa3ea156", "date": "2022-05-06 13:56:58 -0400", "subject": "Update Docker test image for Debian 9.", "body": "A change invalidated the current image which has been causing the d9 test to run longer." }, { "commit": "68a410779ad92a91f6e8bfe504a9acd7fecd06f2", "date": "2022-05-06 12:32:49 -0400", "subject": "Add zNewFmt().", "body": "This replaces strZ(strNewFmt()), making the code simpler and reducing indentation." }, { "commit": "475e7c692d7e54b9417de74cb566c2ccc7158997", "date": "2022-05-06 12:11:04 -0400", "subject": "Clean up dividers in the documentation.", "body": "Dividers were used in some files, but not others, and some had section names (which are hard to maintain) and others did not.\n\nTry to make this more consistent by putting a divider on front of every section, variable block, and wherever else seems appropriate." }, { "commit": "356bc27bf200762148afb7b495b5a54efeea0159", "date": "2022-05-06 11:41:28 -0400", "subject": "Remove key dividers in help.xml.", "body": "The idea was to make this file easier to browse and edit, but in fact it is much easier to just search for the command/option needed.\n\nThe dividers were never applied consistently and at some point we decided to get rid of the comments because they were hard to keep updated. The result was a mix of styles which did nobody any favors." }, { "commit": "65d22e43257d46ff6523ffbdb3af59e30deeaea5", "date": "2022-05-06 11:11:36 -0400", "subject": "Add verify output and verbose options.", "body": "These options allow the user to control how the verify results will be output to the console and log." }, { "commit": "f405fc6ae281eebf4d61abb39963a9830fe2fa2f", "date": "2022-05-06 10:21:20 -0400", "subject": "Backup file bundling documentation.", "body": "Make the feature user visible and add documentation to the user guide." }, { "commit": "4cc0d46d606382e6b00493e73ee401ecfe5b9f3e", "date": "2022-05-06 09:34:39 -0400", "subject": "Fix comment wrapping." }, { "commit": "e70c71049ec1ff3bf1f3e6a1987c82d9b95bce82", "date": "2022-05-06 07:49:23 -0400", "subject": "Use uint8_t for optionResolveOrder.", "body": "This saves a bit of space and should not affect processing speed.\n\nOn MacOS (clang) this unexpectedly reduces the size of the binary by 16kiB but on Linux (gcc) there are no savings at all." }, { "commit": "808f7bf11c367a5be2130ea573c9447d8ea2dda8", "date": "2022-05-05 20:14:13 -0400", "subject": "Replace strNewFmt() with TEST_ERROR_FMT() in command/archive-push module.", "body": "This test was likely written before TEST_ERROR_FMT() existed." }, { "commit": "efe0a39a75ed40f5568b1f5f566fee84d76fe094", "date": "2022-05-05 20:01:02 -0400", "subject": "Use TEST_ERROR_FMT() rather than strNewFmt() in common/lock module.", "body": "These tests were likely written before TEST_ERROR_FMT() existed." }, { "commit": "5089a26633c965c4d9293d3de527a02b42d9b88d", "date": "2022-05-05 18:35:00 -0400", "subject": "Convert strNewFmt() to THROW_FMT() in config/parse module.", "body": "It's not clear why strNewFmt() was used here, but there is no need for it." }, { "commit": "876f3bbd1cd46af7f7d3dc2e9d454b77d51fff80", "date": "2022-05-05 18:15:05 -0400", "subject": "Remove COLON_STR and separator parameter from cfgParseCommandRoleName().", "body": "The separator parameter in cfgParseCommandRoleName() was useless since it was always set to : and COLON_STR did not provide any clarity its the single other usage." }, { "commit": "7ae5478d9840d8b64a1211466e49d2835853ee55", "date": "2022-05-05 12:09:21 -0400", "subject": "Remove most _Z constants.", "body": "Most of the time these were not making the code any clearer.\n\nFor cases where they were used to construct Strings and Buffers, replace with constants.\n\nAlso cleanup unused Buffers and Strings." }, { "commit": "a6b1adb5fd533b4b9aca29646c8acf647d872959", "date": "2022-05-05 11:15:14 -0400", "subject": "Remove extraneous linefeed when writing a lock file.", "body": "Linefeeds are no longer part of the lock file format." }, { "commit": "5f8c9cd66a1b39a47acaddfbc3cf443ae8057519", "date": "2022-05-05 10:19:11 -0400", "subject": "Add ClockError for unexpected clock skew and timezone changes.", "body": "A distinct result code should help debugging of clock skew and timezone issues." }, { "commit": "b6bfd9f99d74b1f5e631cf6462b68342589d2b80", "date": "2022-05-05 09:20:49 -0400", "subject": "Strip extensions from history manifest before showing in error message.", "body": "In cases where clock skew or timezone issues are preventing backup label generation the user could see an error like this:\r\n\r\nnew backup label '20220504-152308F' is not later than latest backup label '20220504-222042F_20220504-222141I.manifest.gz'\r\n\r\nThis will happen if the most recent label is drawn from the history. It is cleaner (and probably less confusing) to strip off the extensions so the user sees:\r\n\r\nnew backup label '20220504-152308F' is not later than latest backup label '20220504-222042F_20220504-222141I'" }, { "commit": "ef672c74adf871f17f6e14f5f0d6ab70e8fa3d52", "date": "2022-05-04 14:53:05 -0400", "subject": "Prevent memContextFree() from freeing memory needed by callbacks.", "body": "The order of callbacks and frees meant that memory needed during a callback (for logging in all known cases) might end up being freed before a callback needed it.\r\n\r\nRequiring callbacks and logging to check the validity of their allocations is pretty risky and it is not clear that all possible cases have been accounted for.\r\n\r\nInstead recursively execute all the callbacks first and then come back and recursively free the context. This is safer and it removes the need to check if a context is freeing so a simple active flag (in debug builds) will do. The caller no longer needs this information at all so remove memContextFreeing() and objMemContextFreeing()." }, { "commit": "d9088b2e2b16a89d29b967187a547976f96061fc", "date": "2022-05-04 12:52:05 -0400", "subject": "Show backup percent complete in info output.", "body": "In the JSON output the percent complete is storage as an integer of the percent complete * 100. So, before display it should be converted to double and divided by 100, or split using integer mod and div.\r\n\r\nNote that percent complete will only be displayed on the host where the backup was executed. Remote hosts will show a backup/expire running with no percent complete." }, { "commit": "20782c88bc8d0842d547bd5f4fecdb292e69d1f7", "date": "2022-05-04 11:55:59 -0400", "subject": "PostgreSQL 15 support.", "body": "PostgreSQL 15 drops support for exclusive backup and renames the start/stop backup commands.\r\n\r\nThis is based on the pgdg-testing repo since beta1 has not been released yet, but it seems unlikely that breaking changes will be made at this point. beta1 should be tagged just before our next release so we'll retest before the release." }, { "commit": "8e849ae85d6ad0ecbd1f34649be4a664712fd40c", "date": "2022-05-04 10:28:39 -0400", "subject": "Add PRs to thread locking on Github actions.", "body": "It also makes sense to lock old PRs. They can be manually unlocked if they are needed for some reason.\n\nAlso add output logging to make it easier to determine if thread locking is completing." }, { "commit": "09b387fccddb658ae93acc6ec8dc490155c4fdea", "date": "2022-05-04 09:24:35 -0400", "subject": "Move issue locking to Github actions.", "body": "The old plugin has been defunct for some time so there are currently a lot of unlocked issues.\n\nRunning this once per week seems sufficient for now. Worst case it can be run manually if it gets behind." }, { "commit": "692fe496bdb5fa6dcffeb9f85b6188ceb1df707a", "date": "2022-05-04 08:22:45 -0400", "subject": "Remove dependency on pg_database.datlastsysoid.", "body": "This column has been removed in PostgreSQL 15. Rather than add a lot of special handling, it seems better just to update all versions to not depend on this column.\r\n\r\nAdd centralized functions to identify the type of database (i.e. system or user) by name and use FirstNormalObjectId when a name is not available.\r\n\r\nThe new query in the db module will still return the prior result for PostgreSQL <= 15, which will be stored in the manifest. This is important to preserve behavior when downgrading pgBackRest. There are no concerns here for PostgreSQL 15 since older versions of pgBackRest won't be able to restore backups for PostgreSQL 15 anyway." }, { "commit": "302e0c09217a2c5c11145e84976b54677f7a544e", "date": "2022-05-03 16:53:29 -0400", "subject": "Remove extra linefeed." }, { "commit": "9a271e925c6239fbd3577f9827c34940e7e43ae4", "date": "2022-05-03 14:34:05 -0400", "subject": "Fix error thrown from FINALLY() causing an infinite loop.", "body": "Any error thrown resets execution to the last setjmp(), which means that parts of the try block need to make sure they don't get run again. FINALLY() was not doing this so if it threw an error it would end up back in the FINALLY() block, where the error would likely be thrown again, causing an infinite loop.\r\n\r\nFix this by tracking the state of FINALLY() and only running it once. This requires cleaning the error stack like CATCH*() and clearing the error like TRY_END() depending on the order of execution." }, { "commit": "b89c568b5fefd1ea0e8319402bab0f4fba6bfa2e", "date": "2022-05-03 10:50:48 -0400", "subject": "Fix obsolete variable naming." }, { "commit": "962990869454b15db058f4037dbc0b1b611fc95e", "date": "2022-05-03 10:13:32 -0400", "subject": "Error on all lock failures except another process holding the lock.", "body": "The archive-get/archive-push commands would not error for, .e.g permissions errors, when attempting to get a lock before launching the async process. Since the async process was not launched there would be no error status file and the user would get a generic failure message. Also, there would be no async log.\r\n\r\nRefactor lockAcquireFile() to throw an error when failOnNoLock = false unless the file is locked by another process. This seems to be the original intent of this parameter and there may have been a mistake when porting from Perl. In any case it looks wrong enough to be considered a bug." }, { "commit": "eb435becb3e4d3dd8ea01b0e8899654a268d3867", "date": "2022-05-02 15:17:34 -0400", "subject": "Exclude mem context name from production builds.", "body": "The mem context name is used to produce clearer debug errors but it has no purpose in production builds.\n\nAlso remove memContextName() and access the struct directly since the name is only used within the common/memContext module.\n\nNote that a few errors that were thrown in production builds (and required the name) are now only thrown in debug builds. In practice we have not seen these errors in production builds due to extensive coverage so it does not seem worth modifying the error to work without the context name.\n\nThis saves some memory, which is worthwhile, but the goal is to refactor Strings and Variants to have their own mem contexts and this change will prevent them from using more memory than they are now, along with other changes that will be coming later." }, { "commit": "0055fa40fee2636fed88699f52e9e67cd2803f71", "date": "2022-05-02 09:45:57 -0400", "subject": "Add user:group to lock permission error.", "body": "This will help debug permissions errors when the lock file cannot be created." }, { "commit": "03c71aa606bc0dc89303b3907e7fcbd83ace79d6", "date": "2022-05-02 08:49:13 -0400", "subject": "Add hint to check the log on archive-get/archive-push async error.", "body": "If this error is thrown rather than a specific error returned from the async process, it means the async process is unable to write the status files for some reason and the only way to get the error is out of the async log.\n\nThis hint includes the exact async log path and name to make finding errors easier." }, { "commit": "126fc99c77998cc65094d92e955347100aa42531", "date": "2022-04-28 14:10:53 -0400", "subject": "Fix leaks in common/json, common/keyValue, and common/variantList.", "body": "This doesn't solve the problem of the variant code making far too many copies, but it at least plugs the leaks.\n\njsonReadVarRecurse() could leak KeyValue and VariantList.\n\nkvDup() leaked object allocations into the calling context.\n\nkvDefault() gets a more efficient return structure.\n\nkvGetList() leaked a Variant into the calling context.\n\nvarLstNewStrLst() leaked object allocations into the calling context. Update varLstDup() to reflect changes made in varLstNewStrLst()." }, { "commit": "96166539cf7df010f9077cd624f24fd77e5a4feb", "date": "2022-04-28 13:33:03 -0400", "subject": "Fix leaked String in cfgParseSize()." }, { "commit": "4872a3f121e71f79a29dd63acad34b7c3537c775", "date": "2022-04-28 12:33:39 -0400", "subject": "Improvements to test harness memory debugging.", "body": "Only set -DDEBUG_MEM for the modules currently being tested rather than globally.\n\nAlso run tests in a temp mem context. Running in the top context can confuse memory accounting when a new context is created in the top context." }, { "commit": "90f939b36f8963272cdfe8d0ed5a50606990ff5e", "date": "2022-04-28 12:31:59 -0400", "subject": "Fix leaks in common/io unit test.", "body": "These leaks make it harder to detect leaks in the core code, so fix them." }, { "commit": "8047e97e31fb133551ad69c7fd170a8a96d188d6", "date": "2022-04-28 12:17:33 -0400", "subject": "Fix leaked String and Variant in harnessPqScriptRun()." }, { "commit": "ceb303f9e2f47d9818919dfd0127e558f2d83495", "date": "2022-04-28 11:46:55 -0400", "subject": "Fix comment typo." }, { "commit": "c463993b4cc96a92c61af298bc95d88129068902", "date": "2022-04-28 11:41:16 -0400", "subject": "Fix leaks in the storage/s3 module.", "body": "storageS3Helper() leaked a few Strings which ended up in a long-lived context.\n\nstorageS3AuthAuto() and storageS3AuthWebId() were cleaned up by their callers but since they are not called often a temp mem context seems better.\n\nstorageS3Request() leaked an HttpRequest.\n\nstorageS3Info() leaked an HttpResponse.\n\nstorageS3PathRemoveInternal() leaked a variety of objects. Fix by freeing some of them and adding a temp mem context.\n\nstorageS3Remove() leaked an HttpResponse object.\n\nstorageWriteS3Part() leaked an HttpResponse object." }, { "commit": "4750bc94dd8279b166233463ff32f2f51fb67e18", "date": "2022-04-28 11:20:31 -0400", "subject": "Fix leaks in the storage/remote module.", "body": "storageRemoteFilterGroup() leaked a number of objects. Use a temp mem context to prevent that.\n\nstorageRemoteProtocolInfoListCallback() leaked a PackWrite.\n\nstorageWriteRemoteFreeResource() leaked a PackWrite." }, { "commit": "c123a6af9f5aee4baea1d9ab202e03df6de319e3", "date": "2022-04-28 11:19:01 -0400", "subject": "Fix leaks in the storage/posix module.", "body": "storagePosixPathCreate() leaked a String.\n\nstoragePosixPathRemoveCallback() leaked a String." }, { "commit": "d89bc6f2d241ae864cb9dcae2f5c6fc3a0e655ee", "date": "2022-04-28 10:53:11 -0400", "subject": "Fix leaks in the storage/gcs module.", "body": "storageGcsAuthToken() memory was being cleaned up by the calling context, but seems better to keep this tidy and add a temp mem context.\n\nstorageGcsRequest() leaked an HttpRequest.\n\nstorageGcsInfo() leaked a number of objects. Use a temp mem context to prevent that.\n\nstorageGcsPathRemoveCallback() leaked an HttpResponse.\n\nstorageGcsRemove() leaked an HttpReponse.\n\nstorageWriteGcsVerify() leaked a number of objects. Use a temp mem context to prevent that.\n\nstorageWriteGcsBlock) leaked an HttpReponse." }, { "commit": "083c93eaa3d54d4594048b37542276fc6f6665fb", "date": "2022-04-28 10:11:15 -0400", "subject": "Reuse Strings in iniLoad().", "body": "Reuse the section/key/value Strings by truncating them instead of creating a new one every time.\n\nAlso add an error for empty sections. This function is only used for loading info files (not config files), which should never contain an empty section." }, { "commit": "bc46d4e37b8c324d30543147eaeb8058184d3543", "date": "2022-04-28 09:50:23 -0400", "subject": "Add cvtZSubNTo*() functions.", "body": "These functions allow conversion from substrings without needing to create a String or a temporary buffer.\n\nhttpDateToTime() no longer requires a temp mem context. Also improve handling of month search to avoid an allocation.\n\nhttpUriDecode() no longer requires a temp mem context.\n\njsonReadStr() no longer requires a temp mem context.\n\npgLsnFromWalSegment() no longer requires a temp mem context.\n\npgVersionFromStr() no longer requires a temp mem context. Also do a bit of refactoring.\n\nstorageGcsCvtTime() no longer leaks six Strings per call.\n\nstorageS3CvtTime() no longer leaks six Strings per call." }, { "commit": "6e18235be8b56eaedd940d7382e09e52ba22ffe9", "date": "2022-04-28 06:52:07 -0400", "subject": "Fix leak in varLstNewStrLst().", "body": "The duplicated list elements were leaked into the calling context." }, { "commit": "aeb7568a0ffeba72f254e46d2aa5d7bcca7c9a19", "date": "2022-04-28 06:48:02 -0400", "subject": "Remove unused header." }, { "commit": "5285b9aa7bdda8c160adde2b7d64e217fd2be123", "date": "2022-04-27 20:18:18 -0400", "subject": "Fix incorrect test param." }, { "commit": "eb65a5674dfa84d6b516490ef8796d44934bb0ef", "date": "2022-04-27 11:40:29 -0400", "subject": "Use OBJ_NEW*() macros in SocketServer object.", "body": "This was missed in ccc255d3 when the TLS server was introduced, probably because work on that commit preceded when the macros were introduced in 475b57c8. It would have been easy to miss in a merge." }, { "commit": "6f2654a5eb0e2cb63a4ff3af187f991817189dca", "date": "2022-04-26 12:53:55 -0400", "subject": "Fix leaks in the storage/azure module.", "body": "storageAzureHelper() leaked a few Strings which ended up in a long-lived context.\n\nstorageAzureNew() failed to make a copy of the endpoint. This worked because storageAzureHelper() leaked the endpoint into the long-lived parent context.\n\nstorageAzureRequest() leaked an HttpRequest.\n\nstorageAzureInfo() leaked an HttpResponse.\n\nstorageAzurePathRemoveCallback() leaked an HttpResponse.\n\nstorageAzureRemove() leaked an HttpResponse." }, { "commit": "be120c746c403b878b6bc7df83b4ffbf6e513f72", "date": "2022-04-26 12:27:51 -0400", "subject": "Refactor and fix leak in storageRepoPathExpression().", "body": "storageRepoPathExpression() could leak a StringList. Also refactor to remove unneeded assignments and create the String one time." }, { "commit": "cca6df872a7861de18f7e69e1df5183acd504681", "date": "2022-04-26 12:09:44 -0400", "subject": "Refactor functions in postgres/interface module and fix leak.", "body": "pgLsnFromWalSegment() leaked two Strings.\n\nRefactor pgLsnRangeToWalSegmentList() to create the StringList in the calling context rather than moving it later." }, { "commit": "a56fa0eb45871dff8b6362d9298290a52c7e6cd9", "date": "2022-04-26 11:59:21 -0400", "subject": "Fix leaks in protocol module.", "body": "These leaks were not a big deal individually and there are generally few protocol objects created, but the leaks ended up in mem contexts that persist for most of execution. It makes sense to keep long-lived contexts as tidy as possible." }, { "commit": "78e912a9320770122808384ae202b89fa6236e89", "date": "2022-04-26 11:20:51 -0400", "subject": "Fix leaks in info module.", "body": "*LoadFileCallback() all leaked between retries.\n\ninfoPgArchiveId() leaked a String." }, { "commit": "36b0a9fa58f4f248fdabf170235f9f624d828d56", "date": "2022-04-26 11:07:05 -0400", "subject": "Fix leaks in db module.", "body": "dbFreeResource() leaked ProtocolCommand.\n\ndbTimeMSec() leaked PackRead." }, { "commit": "78b90e5ad80f87e0c1167701ac40504dc888d0dd", "date": "2022-04-26 11:03:06 -0400", "subject": "Fix leak in configOptionRemote().", "body": "A Pack and String were leaked." }, { "commit": "9a6df398393c116c73dbbf459073271d84c45bec", "date": "2022-04-26 11:00:20 -0400", "subject": "Fix spacing." }, { "commit": "8efb4e1e7f8d806f4688c54052f2325c21f470a9", "date": "2022-04-26 10:58:12 -0400", "subject": "Fix leak in cfgCommandJobRetry().", "body": "A Variant was leaked for each retry record.\n\nAlso remove an extra linefeed." }, { "commit": "a8fed52ecb653229298682280b179fdf92438cdb", "date": "2022-04-26 10:53:13 -0400", "subject": "Fix leak in pckReadStrLst().", "body": "A String would be leaked on each loop iteration." }, { "commit": "d7e45f12a54ea186e66824b9127478eaa1355d32", "date": "2022-04-26 10:49:04 -0400", "subject": "Refactor httpRequestNew().", "body": "Move httpRequestProcess() outside of the object mem context. In case it ever returns a value we don't want that to end up in the object context." }, { "commit": "cd1cf337ff8011f01657ab1baca31927d5d2343d", "date": "2022-04-26 10:45:30 -0400", "subject": "Fix leak in httpHeaderPutRange().", "body": "The range string was leaked." }, { "commit": "07e280199723eef35563ec71ff0c20687d642d9b", "date": "2022-04-26 10:39:28 -0400", "subject": "Fix leak in ioFilterGroupResultAllSet().", "body": "The PackRead containing the filter results was leaked." }, { "commit": "3310decf8e4b09293e3f6b7de7e0212323ad0c81", "date": "2022-04-26 10:36:27 -0400", "subject": "Add missing FUNCTION_TEST*() macros." }, { "commit": "7eed9730aa8af91999e542394d05ef53c1cc7e36", "date": "2022-04-26 10:34:10 -0400", "subject": "Fix leak in cipherBlockNew().", "body": "The string used to look up the cipher type was leaked." }, { "commit": "6eed4125e6297bc6e80c9c11f839ba65979293ee", "date": "2022-04-26 10:29:48 -0400", "subject": "Improved memory management in the common/exit module.", "body": "Refactor so that error detail is only logged in one place. This reduces calls to exitErrorDetail() and LOG_INTERNAL_FMT().\n\nFix minor leaks in exitErrorDetail() and exitSafe()." }, { "commit": "fa6c68cb029e675d9f4b3e4fa48862b6644ecc78", "date": "2022-04-26 10:18:24 -0400", "subject": "Mem allocation cleanup in command/verify module.", "body": "Move the temp mem context out of verifyJobCallback() into verifyBackup() and verifyArchive(). This makes it clearer that verifyJobCallback() allocates no memory and reduces mem usage when both verifyBackup() and verifyArchive() are called.\n\nUpdate verifyErrorMsg() to return zero-terminated strings to save on allocations. The output of this function is used when formatting strings so this is also simpler. Do a similar thing in verifyRender().\n\nAlso fix a minor leak in verifyInfoFile()." }, { "commit": "3f7c8bc9230acaf2dfb0a013303559b54a54c26f", "date": "2022-04-26 10:15:47 -0400", "subject": "Fix object allocations in incorrect mem context in execOpen().", "body": "Object variables were begin allocated in the calling context rather than the object context.\n\nThis is not a live bug because Exec objects are currently created and opened in a long-lived context." }, { "commit": "4dbe76a5bb6e5aae779b2538ac9ce54782f46b8f", "date": "2022-04-26 07:33:57 -0400", "subject": "Fix comment formatting." }, { "commit": "41f9d69edcf671fbda21cb6d87dbca08e48e9976", "date": "2022-04-25 15:38:49 -0400", "subject": "Combine functions in the command/stanza module into one function.", "body": "It is not clear why these were split out, but it probably had something to do with testing before storageList() could return NULL for an empty directory.\n\nAlso remove the tests that depended on a boolean return, which are no longer needed for coverage." }, { "commit": "55a828f999f6fc8a975ea5c2ea9710e2109a5c0a", "date": "2022-04-25 15:03:37 -0400", "subject": "Add temporary mem contexts and fix a leak in the command/restore module.", "body": "restoreRecoveryConf() and restoreRecoveryWriteConf() do enough work to deserve their own memory contexts.\n\nrestoreFilePgPath() was leaking a String every time it was called, which could be a lot.\n\nAlso fix a spacing issue." }, { "commit": "4e7414d48fb3cca45ca66b3eb853a08b6daf2833", "date": "2022-04-25 14:49:08 -0400", "subject": "Add temporary mem contexts in the command/help module.", "body": "These were not really leaks since memory was being freed by the calling function, but these functions do enough work to deserve their own memory contexts." }, { "commit": "fd295f002b720dc1acc5d9523bd442f372394d15", "date": "2022-04-25 14:19:10 -0400", "subject": "Add temporary mem context to removeExpiredBackup().", "body": "This was not really a leak since memory was being freed by the calling function, but this function does enough work to deserve its own memory context.\n\nAlso fixed a doubled semicolon." }, { "commit": "9314be36b161da8128cc4149372f41b89087e844", "date": "2022-04-25 14:14:29 -0400", "subject": "Improve result declaration in backupRegExp().", "body": "No need to set this to NULL when the initial value can be used instead." }, { "commit": "5b5dbe0e7e4d3b3f412ea869487c3018c7a8d67c", "date": "2022-04-25 13:29:43 -0400", "subject": "Fix memory leaks in archivePushDrop().", "body": "A string was leaked for every file processed.\n\nSince the list can be quite large it also makes sense to reset the context occasionally." }, { "commit": "c3b08f71ce03e2b4103f12e1d7c5c05701f02ef3", "date": "2022-04-25 13:26:09 -0400", "subject": "Fix formatting in common/string module." }, { "commit": "40ef64f2be60fb947cf68aae2bb7d510ab36c8ec", "date": "2022-04-25 12:59:46 -0400", "subject": "Fix memory leaks in archivePushFile().", "body": "The errorList is only used when throwing an error and the joined list is not needed after the error is thrown, so put both in the temp mem context." }, { "commit": "774db650868c86c791f2791d02bee129a7669be5", "date": "2022-04-25 12:55:51 -0400", "subject": "Remove useless context switches in archiveGetFile()/archivePushFile().", "body": "These context switches do nothing since the list is already in the prior context." }, { "commit": "582c3dab4cff6883150c9c5de7f7b41d12982da9", "date": "2022-04-25 12:32:33 -0400", "subject": "Add strLstAddSub*() and strLstAddSubZ*() functions.", "body": "These help with readability and remove a cause of leaks." }, { "commit": "ff45f463cf4d4521df5fd69b48bd553d862f0dc1", "date": "2022-04-25 11:58:30 -0400", "subject": "Use strLstAddZ() instead of strLstAdd() where possible.", "body": "Using STRDEF() to convert the zero-terminated string to a String has no performance advantage but generates more code." }, { "commit": "7900660d3a12573db8a68bc7df46d7552dae54d1", "date": "2022-04-25 11:47:43 -0400", "subject": "Add strLstNewFmt().", "body": "Simplifies adding a formatted string to a list and removes a common cause of leaks." }, { "commit": "3475514b61fec52a5bd9e7fa73d38d5be612a9ad", "date": "2022-04-25 10:57:36 -0400", "subject": "Fix memory leak in archiveAsyncErrorClear().", "body": "A string was leaked on each call." }, { "commit": "23d645b5e7f1069296f1a51165412049e1dccbdc", "date": "2022-04-25 10:52:13 -0400", "subject": "Fix outdated comment." }, { "commit": "699f15dd2b227ad13ba313fa549c32938bd6cf2b", "date": "2022-04-25 09:24:00 -0400", "subject": "Remove THIS_MEM_CONTEXT() macro.", "body": "objMemContext(this) performs the same task and is easier to read.\n\nMost instances of this macro were removed by 6e7be3c0." }, { "commit": "6e7be3c0525782cb58e0eb5046f1385e4eb8b1df", "date": "2022-04-25 09:12:25 -0400", "subject": "Add MEM_CONTEXT_OBJ_*() macros.", "body": "These provide a standard way to switch to an object mem context.\n\nUpdate the two different patterns that were used before to the new macros." }, { "commit": "45c3f4d53c7e5198d4d49109398326fecd282bec", "date": "2022-04-25 09:06:26 -0400", "subject": "Improve JSON handling.", "body": "Previously read/writing JSON required parsing/render via a variant, which add many more memory allocations and loops.\r\n\r\nInstead allow JSON to be read/written serially to improve performance and simplify the code. This also allows us to get rid of many String and Variant constant which are no longer required.\r\n\r\nThe goal is to be able to read/write very large (e.g. gigabyte manifest) JSON structures, which would not be practical with the current code.\r\n\r\nNote that external JSON (GCS, S3, etc) is still handled using variants. Converting these will require more consideration about key ordering since it cannot be guaranteed as in our own formats." }, { "commit": "58f24568f5bf578e8ad0f5e4866bf2ede8b45d46", "date": "2022-04-25 08:25:48 -0400", "subject": "Add ASSERT_PARAM() macro.", "body": "Allows adding a parameter to a function that is used only for assertions." }, { "commit": "1e2b545ba47d66760cf4c6df65c6325bb6404bb2", "date": "2022-04-24 19:19:46 -0400", "subject": "Require type for FUNCTION_TEST_RETURN*() macros.", "body": "This allows code to run after the return type has been generated in the case where it is an expression.\n\nNo new functionality here yet, but this will be used by a future commit that audits memory usage." }, { "commit": "91bbe810592c8b57079230d0bf15720d74c705fa", "date": "2022-04-22 09:14:12 -0400", "subject": "Add SIZE_OF_STRUCT_MEMBER() macro.", "body": "Gets the size of a struct member, which requires a bit of additional syntax." }, { "commit": "a2eee156b5a85237d9b9ba7896b237ec4f216da0", "date": "2022-04-21 18:23:17 -0400", "subject": "Fix instances where STRDEF() was used instead of STR().", "body": "In practice this didn't cause problems because the string buffer was still valid and strSize() was not being called." }, { "commit": "627921c72ab53a88519902c17450cedfa9dbda7a", "date": "2022-04-21 08:35:31 -0400", "subject": "Add STR_SIZE() macro.", "body": "This allows efficiently creating strings where the length is already known and simplifies the STR() and STRDEF() macros." }, { "commit": "1ce613a24d4de1c219d2b7d844cf7e88649887fc", "date": "2022-04-21 08:34:03 -0400", "subject": "Use STRDEF() instead of STR() in command/help module.", "body": "STRDEF() is more efficient since this is a constant string." }, { "commit": "4daddebaca6b4b1861778bc2b09d265302c05ed1", "date": "2022-04-21 08:07:22 -0400", "subject": "Add ASSERT_DECLARE() macro.", "body": "Declare variables that will be used by later assertions with the goal of making them easier to read and maintain.\n\nThis is particularly useful for variables that are used more than once and require a lot of syntax to extract." }, { "commit": "e18b70bf555de24ba38d7c51f9d0ffbe8a02a355", "date": "2022-04-21 07:45:59 -0400", "subject": "Allow *RETURN*() macros to accept struct initializers.", "body": "Struct initializers look like multiple parameters in a macro so use __VA_ARGS__ to reconstruct them." }, { "commit": "ea4d73f375558c237fce696f20c19d1efef00bc1", "date": "2022-04-20 19:56:26 -0400", "subject": "Fix ordering of backup-lsn-stop field in command/restore unit test.", "body": "All fields should be alphabetical. Currently the read code is tolerant of this, but that will not always be the case.\n\nFields are always written alphabetically so this is just a test issue introduced by d8d41321." }, { "commit": "cb7a5f1ef3bbfe0a26e3cd718d9635280f2e9b83", "date": "2022-04-20 19:49:23 -0400", "subject": "Add JSON error when value does not parse in Ini object.", "body": "If the JSON value fails to parse it is helpful to have the error message, at least for debugging." }, { "commit": "da6b4abc58674902196fe549586c16094dc37e38", "date": "2022-04-20 19:41:28 -0400", "subject": "Handle missing archive start/stop in info/info backup unit test.", "body": "This is not a very realistic case since archive start/stop are always written, but it appears in many other unit tests so it should also be tested here." }, { "commit": "d897bf1ec2c74e46c56c31c6ba2ca910394ce0e5", "date": "2022-04-20 19:36:33 -0400", "subject": "Add size to info/manifest unit test.", "body": "This prevents the check from being order dependent." }, { "commit": "d6f466be2bd1b68d45f109f0332783b784838ab7", "date": "2022-04-20 19:31:35 -0400", "subject": "Remove unnecessary mem contexts in the Manifest object.", "body": "The correct context is set by the various *Add() functions so these are not needed and cause leaks, though the leaks will only be noticeable in cases where there are a lot of page checksum errors." }, { "commit": "da9f2618528e9bd4b5538ff81cc359172a36af7b", "date": "2022-04-20 14:09:49 -0400", "subject": "Add FUNCTION_TEST_NO_RETURN() macro.", "body": "This is required for the (currently) single place where a function with test FUNCTION_TEST*() macros does not return.\n\nThis allows return to be added to the FUNCTION_TEST_RETURN_VOID() macro, which means return no longer needs to be added when returning from a function early." }, { "commit": "c304fafd456f67d11b0e9e7a8f399732716cf216", "date": "2022-04-20 08:36:53 -0400", "subject": "Refactor PgClient to return results in Pack format.", "body": "Packs support stronger typing than JSON and are more efficient. For the small result sets that we deal with efficiency is probably not very important, but this removes another place where we are using JSON instead of Pack.\r\n\r\nPush checking for result struct (e.g. single row) down into PgClient since it has easy access to this information rather than needing to parse the result set to find out.\r\n\r\nRefactor all code downstream that depends on PgClient results." }, { "commit": "e699402f99f70819bd922eb6150fbe1b837eca0d", "date": "2022-04-19 18:07:11 -0400", "subject": "Remove extra linefeed." }, { "commit": "b7fccaf9949f15ea1a00abd434e765c9c4d3e2f1", "date": "2022-04-18 14:08:53 -0400", "subject": "Refactor remote storage protocol to use Packs instead of JSON.", "body": "Packs are more efficient and strongly typed so they make more sense for the protocol." }, { "commit": "cfd6c7ceb490d8c2766fd91a4719bdf9f6e4f110", "date": "2022-04-18 12:14:22 -0400", "subject": "Use specific integer types in postgres/client and db unit tests.", "body": "This will work better once we are able to transmit the results with stronger typing.\n\nAlso remove int2 which was not being used." }, { "commit": "9751ddc4f80928de4b42bc56c54731afa4ff16aa", "date": "2022-04-18 11:53:31 -0400", "subject": "Update postgres/client unit test to conform to current patterns.", "body": "This includes adding test titles and using constants for query and error values that repeat." }, { "commit": "bc5f6fac34fefd0595eee87f250dc9abef06d191", "date": "2022-04-18 10:47:44 -0400", "subject": "Update postgres/client unit test for changes in libpq.", "body": "There have been some behavioral changes in libpq which require changes to the test.\n\nAlso update the instructions since it is now a bit easier to run against a real cluster." }, { "commit": "214ee9eb0e8398b1871a53441f4b926eb194d765", "date": "2022-04-17 09:41:22 -0400", "subject": "Fix URL for apt.p.o archives.", "body": "A new archive repo was created in March of 2020: https://www.df7cb.de/blog/2020/apt-archive.postgresql.org.html" }, { "commit": "d103dd6238dffb5ddc3d5366c674278a67426f25", "date": "2022-04-14 20:34:42 -0400", "subject": "Return stats as a JSON string rather than a KeyValue object.", "body": "There is no need to process the stats so a KeyValue is overkill.\n\nAlso remove the performance tests that check the stat totals since this is covered in the unit tests." }, { "commit": "e1ce731f8ae5d1aea009bc369fd157cddb44e059", "date": "2022-04-14 19:37:03 -0400", "subject": "Add test for protocol greeting when a field is missing.", "body": "A missing field and a NULL field are not exactly the same so it seems best to test both.\n\nBecause of the way KeyValue objects work the error is the same, but that will not always be true." }, { "commit": "8a29d56f3cbf37166f3722ac7ffe18841e9e57f3", "date": "2022-04-14 19:18:00 -0400", "subject": "Update comment in pack module to cover a more common use case.", "body": "The KeyValue object is actively being removed so this is no longer the best example.\n\nInstead use an example that should outlive the KeyValue object." }, { "commit": "aeecd07ad87fafb34087f0df9d7c7413f749194c", "date": "2022-04-14 18:29:54 -0400", "subject": "Fix reported error line number when ini key length is zero.", "body": "The line number was one less than it should have been, which could cause some confusion.\n\nSince this only affected ini files with JSON values, which are always written programmatically, there is almost zero chance this has ever been a problem in the field." }, { "commit": "47f8e11889ab8c05e1e548c07e6a338b69745c86", "date": "2022-04-14 08:13:39 -0400", "subject": "Update FreeBSD and MacOS images for Cirrus CI.", "body": "This fixes fe1ac210. Apparently FreeBSD 12.2 went EOL and the image was immediately broken.\n\nAlso add FreeBSD 13.0 and update MacOS to Monterey." }, { "commit": "bcbac8a067e84a48e0849c5336f21044fec89df5", "date": "2022-04-11 17:54:18 -0400", "subject": "Add static keywords.", "body": "This is more efficient than pushing these variables onto the stack." }, { "commit": "fa40bcdc5cf9e6fc6f46236df258d3fa536ccdc7", "date": "2022-04-11 14:08:16 -0400", "subject": "Throw error when unable to read lock process.", "body": "Previously the process id was skipped if it did not exist. Instead, throw an error and handle the errors in downstream code.\r\n\r\nThis was probably ignored at some point to provide backward-compatibility, but that is no longer required, if it ever was." }, { "commit": "fe1ac210bba9439d9a4baa4adbadd1413c61e2f5", "date": "2022-04-11 13:17:54 -0400", "subject": "Disable FreeBSD builds on Cirrus CI.", "body": "This build has started breaking with the following error:\n\ncd .. && perl ${CIRRUS_WORKING_DIR}/test/test.pl --no-gen --make-cmd=gmake --vm=none --vm-max=2 --no-coverage --no-valgrind --module=command --test=backup\n2022-04-11 17:11:53.034 P00 INFO: test begin on amd64 - log level info\n2022-04-11 17:11:53.107 P00 INFO: configure build\nld-elf.so.1: /usr/local/lib/perl5/5.32/mach/CORE/libperl.so.5.32: Undefined symbol \"strerror_l@FBSD_1.6\"\n\nDisable the build to unstick the pipeline until this can be fixed." }, { "commit": "15021a0e976c1a8eadebf642dfff293df0fed54a", "date": "2022-04-09 18:29:57 -0400", "subject": "Fix whitespace." }, { "commit": "79b204166394295e40e7ceddc359368c5770db26", "date": "2022-04-08 15:55:41 -0400", "subject": "Add lockRead*() functions for reading locks from another process.", "body": "Sometimes we need to read a lock from another process. This was done two different ways and in the case of cmdStop() was definitely hacky.\r\n\r\nCentralize the logic to make it easier to read the locks for another process. This will also make it easier to add new lock data." }, { "commit": "aad7171940afdb221a03825bee6eb2f1827289f1", "date": "2022-04-08 15:00:20 -0400", "subject": "Suppress existing WAL warning when archive-mode-check is disabled.", "body": "When archive-mode-check is disabled and archive-push is running from multiple hosts, it is very likely that the file will already exist with the same checksum, so disable the warning.\r\n\r\nHowever, if the checksums do not match, an error will still be thrown." }, { "commit": "4f543a4d67e0c18c8eccdd864a32003d45df7426", "date": "2022-04-08 11:07:26 -0400", "subject": "Handle NULL path in TEST_STORAGE_LIST when remove is specified.", "body": "Using the path variable directly resulted in a path with (null) in it, which caused the remove to fail.\n\nThe pathFull variable already exists for this purpose so use it." }, { "commit": "571dceefec5c64da2b2d79dd7e471fe0262b916a", "date": "2022-04-07 19:00:15 -0400", "subject": "Add LENGTH_OF() macro.", "body": "Determining the length of arrays that could be calculated at compile time was a bit piecemeal, with special macros used sometimes and with the math done directly other times.\n\nThis macro makes the task easier, uses less space, and automatically adjusts when the type changes." }, { "commit": "8be11d32e43bc889c4889b28f88fe20a0c4cfd60", "date": "2022-04-07 11:44:45 -0400", "subject": "Replace strCatFmt() with strCat()/strCatZ() where appropriate.", "body": "Most of these looked like copy/paste from a prior required strCatFmt() call.\n\nThere is no issue here since strCatFmt() works the same in these cases, but using strCat()/strCatZ() is more efficient." }, { "commit": "cff147a7d257e270df8fee184fdc3684347be330", "date": "2022-04-06 14:45:51 -0400", "subject": "Add default for boolean options with unresolved dependencies.", "body": "If a boolean option had an unresolved dependency then the value would be NULL, which meant the dependency would need to be checked in the code to avoid an error. For example, cfgOptionBool(cfgOptOnline) needed to be checked before it was safe to call cfgOptionBool(cfgOptArchiveCheck).\r\n\r\nAllow a default for boolean options when they are unresolved to simplify the code. This makes using the options easier and less prone to error. Not all boolean options get a dependency default in this commit, but more may be added in the future." }, { "commit": "acc9f3b72a481f64f5362d862959f300eb2aa63b", "date": "2022-04-06 10:04:08 -0400", "subject": "Move cfgParseOptionalFilterDepend() and add comment block." }, { "commit": "5dba0d6e9b9129c3cb7f3a4bc46da57544f36e73", "date": "2022-04-05 18:42:19 -0400", "subject": "Set option-archive-copy flag in backup.manifest to false when offline.", "body": "In offline mode the pg_wal directory is copied, but that is not the same as archive-copy, which copies the exact set of WAL required from the archive.\n\nThis flag is purely for informational purposes so there is no live bug here, but the prior behavior was certainly misleading." }, { "commit": "54b4187527287bf3dd8ae158329f0feb44eebed8", "date": "2022-04-05 13:14:42 -0400", "subject": "Show Docker output when building containers if --log-level=detail.", "body": "This helps with debugging and monitoring container builds." }, { "commit": "d8d4132118edd7f93a5b221478c85c2d213c39dd", "date": "2022-04-05 11:59:12 -0400", "subject": "Auto-select backup for restore command --type=lsn.", "body": "For PITR with --type=lsn, attempt to auto-select the appropriate backup set based on the --target LSN provided. Pick the most recent backup where backup-lsn-stop is less than or equal to the provided LSN." }, { "commit": "08d9e269c6f03e7797584e90c53eb0743b787f60", "date": "2022-04-05 09:14:56 -0400", "subject": "Refactor target type checking for clarity in restore module.", "body": "This avoids using targetTime != 0 as an indicator that a time target was selected." }, { "commit": "f5fdab2989140630a62a4ef0d4c7fc35d789f967", "date": "2022-03-31 17:09:50 +0200", "subject": "Fix typo in db-timeout help." }, { "commit": "c222ce1a5f6f813188b7e87a3838d6dce656e0ff", "date": "2022-03-25 08:52:41 -0600", "subject": "Fix tls-server-auth example and add clarifications." }, { "commit": "d4954fcbf49790475d26c90e951ac76660cef1e9", "date": "2022-03-25 07:52:16 -0600", "subject": "Fix comment typo." }, { "commit": "f60ec5055a633308422797902e9487085c862850", "date": "2022-03-24 18:43:43 -0600", "subject": "Cleanup output to stderr in unit tests.", "body": "The unit tests were ignoring stderr but nothing being output there was important. Now a test will fail if there is anything on stderr.\n\nThis makes it easier to work with -fsanitize, which outputs to stderr." }, { "commit": "50ee4b19fec0b6bc888e4ff6603166365ce88233", "date": "2022-03-24 17:55:38 -0600", "subject": "Align checksum page error list in manifest file packs.", "body": "This was left unaligned on purpose to save space but the sanitizer did not like it. Since this field is seldom used go ahead and align it to make the sanitizer happy.\n\nAlso add some macros to make working with alignment easier.\n\nFound with -fsanitize=undefined." }, { "commit": "3dd796045160c4d4dbf70ec2800ad483867629f1", "date": "2022-03-24 15:59:41 -0600", "subject": "Fix incorrect reference to stanza-update in the user guide.", "body": "This should be stanza-upgrade. Also fix in the git history cache since the comment was copied from the user guide." }, { "commit": "14016a86e7d37030ae934a004cad70888b539b33", "date": "2022-03-24 13:13:35 -0600", "subject": "Check that sha1 checksum is not empty in manifestFileUpdate().", "body": "The manifest test module was setting a blank value here and causing a stack overflow because memcpy() is used instead of strcpy().\n\nThis was really just a test issue but add an assert just in case the same were to happen in production code.\n\nAlso update a bogus checksum in the integration tests to the correct length to avoid running afoul of the assert.\n\nFound with -fsanitize=address." }, { "commit": "4e5ac11517022f445f4ec078cc30bd2ce3aa6d15", "date": "2022-03-24 12:56:26 -0600", "subject": "Fix incorrect struct type in list initialization.", "body": "This looks like a copy-paste error.\n\nThe code is only run during development so this is not a live issue.\n\nFound with -fsanitize=address." }, { "commit": "75b26319aea4493d8b9624a61795f8128d50c3b5", "date": "2022-03-24 12:26:09 -0600", "subject": "Use strNewZ() in cases where STRDEF() assignment goes out of scope.", "body": "If a variable assigned with STRDEF() is referenced out of scope of the STRDEF() assignment then the value is undefined.\n\nLuckily most of the instances are in tests but there is one in the core code. It is not clear if this is a live bug or not but it certainly needs to be fixed.\n\nFound with -fsanitize=address." }, { "commit": "edf6c70baa97ad9c9045c01df4f587880a78cdc6", "date": "2022-03-24 11:00:51 -0600", "subject": "Prevent signed integer overflow in cfgParseSize().", "body": "If the value and multiplier were large enough then the return value could overflow unpredictably.\n\nCheck the value to make sure it will not overflow with the current multiplier.\n\nIt would be better to present an \"out of range\" error to the user rather than \"is not valid\" but it doesn't seem worth the effort since the error is extremely unlikely.\n\nFound with -fsanitize=undefined." }, { "commit": "ccbe2a1f70b3ec0a328c7f5d0f5053ef3de1b543", "date": "2022-03-24 09:32:18 -0600", "subject": "Do not pass NULL to memcpy() in Buffer/String objects.", "body": "glibc and others seem tolerant of this but the behavior is undefined.\n\nFound with -fsanitize=undefined." }, { "commit": "98792b1b0cd084ff90343e65b64526dfede396e7", "date": "2022-03-24 09:22:05 -0600", "subject": "Do not pass NULL to bsearch()/qsort() in List object.", "body": "glibc and others seem tolerant of this but the behavior is undefined.\n\nFound with -fsanitize=undefined." }, { "commit": "333ef84606cb1597e09a324c81759a2a222e3bf4", "date": "2022-03-23 21:47:56 -0600", "subject": "Use bufUsed() instead of struct member in Buffer object.", "body": "bufUsed() is an inline function so it is just as efficient and should also be easier to read and maintain." }, { "commit": "424008d293661dfb1a167cfec2697def48057447", "date": "2022-03-23 10:41:36 -0600", "subject": "Allow files that become zero-length after the backup manifest is built.", "body": "It is possible that a file will be be truncated to zero-length after the backup manifest has been built. We could build logic into backupFile() to handle this case but it is hard to test well because of the race condition so tests would need to written directly against backupFile() and backupJobResult(). It hardly seems worth all that effort for a condition that occurs rarely, if ever.\n\nInstead just remove the manifest check and add tests to restore to make sure it handles bundled zero-length files correctly. Logging will show that the file was bundled so if it happens a lot (which seems very unlikely) then we can think about an alternate implementation." }, { "commit": "fe9fd2ff2d778b86c0df159aabe7644f18f6479f", "date": "2022-03-22 09:02:33 -0600", "subject": "Disable repo-hardlink option when repo-bundle option is enabled.", "body": "Hardlinking will not work with bundles because files are not stored individually." }, { "commit": "7afaac0a3d0e43e5598d1abf548559dbe6329363", "date": "2022-03-22 08:35:34 -0600", "subject": "Allow repo-hardlink option to be changed after full backup.", "body": "This rule was added because there were not sufficient tests to demonstrate that the repo-hardlink option could be changed in a backup set.\r\n\r\nRemove the restriction and add/update tests to show that it works.\r\n\r\nThis is necessary now because bundling requires that hardlinking be disabled. Rather than add code complexity, it seems better just to address this limitation." }, { "commit": "5ae84d5e474e95dcc00c174aa5111815a5a70ba5", "date": "2022-03-22 07:50:26 -0600", "subject": "Improve path validation for repo-* commands.", "body": "Check for invalid path in repo-* commands. Perform path validation and throw an error when appropriate. Path may not contain '//'. Strip trailing '/' from path. Absolute path must fall under repo path." }, { "commit": "21cef09dfd03c313531dc66ba9556164ae2835ab", "date": "2022-03-16 11:02:29 -0600", "subject": "Add AWS IMDSv2 support.", "body": "IMDSv2 provides additional security to prevent instance metadata from being read by an attacker.\r\n\r\nAll AWS instances should provide IMDSv2 but still fail back to IMDSv1 if the IMDSv2 token request fails. This is in case there are any services outside AWS that are emulating IMDSv1 but have not implemented IMDSv2." }, { "commit": "2c96327e654ab634ee4077a39d6d20a7eae1065f", "date": "2022-03-15 17:55:48 -0600", "subject": "Remove extraneous double spaces in code and comments." }, { "commit": "3f66f42ef9605a83c356e4c6eb3ef47bcb12c30d", "date": "2022-03-14 17:49:52 -0600", "subject": "Rename bundle-* options to repo-bundle-*.", "body": "It seems best for these to be repo options so they can be configured per repo, rather than globally.\r\n\r\nAll clarify usage for repo-bundle-size and repo-bundle-limit." }, { "commit": "7c9208ba856a71833dc7d8f1cf8cb2b4bb4b7414", "date": "2022-03-11 10:10:02 -0600", "subject": "Improve error message for invalid repo-azure-key.", "body": "Check that repo-azure-key is valid base64 when repo-azure-key-type = shared." }, { "commit": "0054677147121967889ead7feab0959fb19e39a6", "date": "2022-03-09 15:34:15 -0600", "subject": "Add bundle logging to backup command.", "body": "This was added to the restore command so add it to the backup command as well." }, { "commit": "dca6da86bfe84a0cda004df1140d125db1630c90", "date": "2022-03-09 15:03:28 -0600", "subject": "Optimize restore command for file bundling.", "body": "Since files are stored sequentially in a bundle, it is often possible to restore multiple files with a single read. Previously, each restored file required a separate read. Reducing the number of reads is particularly beneficial for object stores, but performance should benefit on any file system.\r\n\r\nCurrently if there is a gap then a new read is required. In the future we might set a limit for how large a gap we'll skip without starting a new read." }, { "commit": "f7ab002aa74dd2ca277123f7500018ae68a04c74", "date": "2022-03-08 12:18:23 -0600", "subject": "Improve stop command to honor stanza option.", "body": "Improve the stop command, when force and stanza options are specified, to terminate only processes holding lock files for the given stanza. Prior to these changes, termination of all processes holding lock files regardless of stanza occurred." }, { "commit": "514137040e60e8fff0e7f42f43b493113abae5ff", "date": "2022-03-08 08:23:31 -0600", "subject": "Add limit parameter to ioCopyP().", "body": "Allows the number of bytes copied to be limited." }, { "commit": "166039c0da1875581a08d41ab8a44adf95fa6db1", "date": "2022-03-08 08:09:58 -0600", "subject": "Fix example for repo-gcs-key-type option in configuration reference.", "body": "This looks like a copy-paste error from another option." }, { "commit": "91b718bd4f89157c7401e3904bb171387b2e8a3e", "date": "2022-03-06 14:58:43 -0600", "subject": "Fix comment typo." }, { "commit": "330e19900ee3415d920680cf6e6856bf5e8e9a69", "date": "2022-03-06 13:01:24 -0600", "subject": "Increase precision of percent complete logging for backup and restore.", "body": "For very large backups only getting an update per percent may not be often enough.\r\n\r\nAdd hundredths to the percent complete logging to provide more timely information." }, { "commit": "be731f825432c738571f4c055f1b9d42d3fa18cf", "date": "2022-03-06 12:03:57 -0600", "subject": "Fix incorrect comment.", "body": "This appears to have been blindly copied from the archive-get command." }, { "commit": "8f23b46b4be9a0ce55e0c2d13be0edfd5bdc5349", "date": "2022-03-06 11:57:20 -0600", "subject": "Replace percentage and size with a constant in restore test logs.", "body": "Checking percentage and size in every test can cause quite a bit of churn when changes are made.\n\nFollow the example of the backup tests and replace percentage and size after the few tests to reduce churn." }, { "commit": "4d2fef1c37d357965f0ffb9af0d7489f3be1371b", "date": "2022-03-06 11:48:22 -0600", "subject": "Remove redundant restoreFile() test and improve coverage.", "body": "These tests were written before the restore command was fully migrated to C so many of them have become redundant.\n\nIn the cases were they still provide coverage, add tests to synthetic restores to replace them. In general, these higher level tests provide better coverage than poking at the restoreFile() function directly." }, { "commit": "009d33aa8ebbcd00a85392e1424e8f078952e2f5", "date": "2022-03-06 11:17:01 -0600", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "c242b966e0352505ee8389fcebcfbde48b2a55d4", "date": "2022-03-06 11:14:01 -0600", "subject": "Begin v2.39 development." }, { "commit": "5249b89a2e94fdbb45461faaf2e1b0304b986ac7", "date": "2022-03-06 10:30:59 -0600", "subject": "v2.38: Minor Bug Fixes and Improvements" }, { "commit": "59a5373cf85e4ad0057b592bfe855495384dfcbd", "date": "2022-03-02 11:38:52 -0600", "subject": "Handle TLS servers that do not close connections gracefully.", "body": "Some TLS server implementations will simply close the socket rather than correctly closing the TLS connection. This causes problems when connection: close is specified with no content-length or chunked encoding and we are forced to read to EOF. It is hard to know if this is a real EOF or a network error.\r\n\r\nIn cases where we can parse the content and (hopefully) ensure it is correct, allow the closed socket to serve as EOF. This is not ideal, but the change in 8e1807c means that currently working servers with this issue will stop working after 2.35 is installed, which seems too risky." }, { "commit": "f1bdf3e04b07dc9a2eaa1fdbe5278cc18371e868", "date": "2022-03-02 10:59:06 -0600", "subject": "Add aliveness check to systemd service configuration.", "body": "If the pgbackrest service service fails to start then the user will get an error. However, retries will continue in the background." }, { "commit": "a66ec8d549b8f1f7b38682ad90281f1fcbb2748a", "date": "2022-03-02 10:22:15 -0600", "subject": "Revert PGDG yum repo workaround for aarch64.", "body": "da0f3a855 used a workaround to get the documentation building on aarch64 but recent changes to the PGDG yum repo have broken this workaround. Installing the regular way still doesn't work, either.\n\nReverting for now to get the CI pipeline working again." }, { "commit": "fb5051fde74bcfde85fa3b9a452a1df693e79db9", "date": "2022-02-26 13:50:30 -0600", "subject": "Use vagrant user in the Docker container.", "body": "This is a bit of legacy from the current Vagrant environment used to do the release, but since it is not as easy to change the user in Vagrant, just make the Docker environment conform.\n\nThis allows documentation to be built in a Vagrant environment (or any environment with the same user name) and to be deployed in a Docker environment." }, { "commit": "f716e98ad0e9eda94da973211e5f54e88c2c1aed", "date": "2022-02-26 11:31:52 -0600", "subject": "Suppress errors when there is stderr output for Docker builds.", "body": "Docker outputs build info to stderr even when the build is successful. This seems to be especially true on Mac M1.\n\nContainerTest.pm already does this suppression so add it the other places where containers are built." }, { "commit": "53de3e3aeb71f8209f56167941df35bbfcb7cdc9", "date": "2022-02-26 11:17:05 -0600", "subject": "Move repo options in TLS documentation to the global section.", "body": "These options were mistakenly added to the stanza section, which works in certain cases, but is not best practice." }, { "commit": "98d525dba43ba259ff7989746b8e5be3382c189d", "date": "2022-02-25 14:03:09 -0600", "subject": "Add FAQ explaining WAL archive suffix." }, { "commit": "b33cabe08c33e29eb8aef65dd46b2c1f3cf2d186", "date": "2022-02-25 10:51:40 -0600", "subject": "Allow case-insensitive matching of HTTP connection header values.", "body": "The specification allows values for the connection header to be case-insensitive. See https://www.rfc-editor.org/rfc/rfc7230#section-6.1." }, { "commit": "632071232344a1160ef932d6e0d65eb3149ed226", "date": "2022-02-23 13:53:02 -0600", "subject": "Automatically strip trailing slashes for repo-ls paths.", "body": "Trailing slashes in at least some of the repository storage types were preventing repo-ls from displaying any content (presumably due to storage-specific behavior).\r\n\r\nSince the path with the slash should be equivalent to the path without the slash, just remove it if provided by the user." }, { "commit": "53f1b2520434221c01caf92e052403a28c455c1b", "date": "2022-02-23 13:17:14 -0600", "subject": "Improve validation of zero pages.", "body": "Checking that pd_upper == 0 is not enough since this field may be corrupted. Still use pd_upper as a quick check, but when it is zero proceed to check the rest of the page to ensure it is also all zeroes." }, { "commit": "9eec98c61302121134d2067326dbd2cd0f2f0b9c", "date": "2022-02-23 12:05:53 -0600", "subject": "Retry on page checksum validation failure during backup.", "body": "Rather than attempting to filter page checksum failures by LSN, just retry when there is a page checksum failure. If the page has not changed since the last read report it as an error. If the page has changed, then PostgreSQL must be modifying the page so we can ignore the error because a full page write (and possibly updates) will be in the WAL.\r\n\r\nAlso remove tests made redundant by the test merge in b4897077." }, { "commit": "5c332a0b100d5a5180b3d95bd90d4d7b521b42db", "date": "2022-02-23 11:23:32 -0600", "subject": "Fix release note attribution in cacfdd94 and f798458e." }, { "commit": "f798458e1d18fc8a866b49a4cfa0cb73536cbd41", "date": "2022-02-23 10:29:17 -0600", "subject": "Disconnect help.auto.c from build-help in Makefile.", "body": "When there was an issue with the system library path during building, the build-help rule would fail during executing ./build-help with the effect that main.c wouldn't build.\r\n\r\nBreak out help.auto.c generation from the build-help stage to allow it to be re-executed when the library path has been corrected." }, { "commit": "67bdf07e6969f8160347e895454bf729c9165884", "date": "2022-02-23 10:26:39 -0600", "subject": "Add XML to invalid XML error message.", "body": "There have been cases where pgBackRest has failed on invalid XML but it is not possible to determine what was wrong with the XML.\n\nThis will only work for XML up to about 8KiB (which is the error message limit) but it should work in most cases." }, { "commit": "10038db9c9f2b4753c3821e0b246fa8626527184", "date": "2022-02-23 09:14:27 -0600", "subject": "Add archive-missing-retry option.", "body": "Retry a WAL segment that was previously reported as missing by the archive-get command. This prevents notifications in the spool path from a prior restore from being used and possibly causing a recovery failure if consistency has not been reached.\r\n\r\nDisabling this option allows PostgreSQL to more reliably recognize when the end of the WAL in the archive has been reached, which permits it to switch over to streaming from the primary. With retries enabled, a steady stream of WAL being archived will cause PostgreSQL to continue getting WAL from the archive rather than switch to streaming.\r\n\r\nWhen disabling this option it is important to ensure that the spool path for the stanza is empty. The restore command does this automatically if the spool path is configured at restore time. Otherwise, it is up to the user to ensure the spool path is empty." }, { "commit": "cacfdd94d74307285b1b590ec0977d7db3142d0d", "date": "2022-02-23 08:01:38 -0600", "subject": "Fix typos in help and release notes." }, { "commit": "e2eb3ebacce18e9c0c2f74457a646142a0eace3d", "date": "2022-02-20 17:45:41 -0600", "subject": "Most recent news for postgresql.org." }, { "commit": "e6e1122dbcf5e667d683295f7e7e45de4bbf56bd", "date": "2022-02-20 16:45:07 -0600", "subject": "Pass file by reference in manifestFileAdd().", "body": "Coverity complained that this pass by value was inefficient:\n\nCID 376402: Performance inefficiencies (PASS_BY_VALUE)\nPassing parameter file of type \"ManifestFile\" (size 136 bytes) by value.\n\nThis was completely intentional since it gives us a copy of the struct that we can change without bothering the caller. However, updating fields is fine and may benefit the caller at some future data, and in any case does no harm now.\n\nAnd as usual it is easier not to fight with Coverity." }, { "commit": "b4897077937ee4571ba719276a44d5db0a75510e", "date": "2022-02-18 17:50:05 -0600", "subject": "Move command/backup-common tests in the command/backup module.", "body": "As much as possible it is better to get coverage with more realistic tests. Merging these modules will allow the page checksum code to be covered with real backups." }, { "commit": "efc09db7b9ece6e7b7f92538d56d6ab7b9798f8f", "date": "2022-02-17 07:25:12 -0600", "subject": "Limit files that can be bundled.", "body": "Limit which files can be added to bundles, which allows resume to work reasonably well. On resume, the bundles are removed and any remaining file is eligible to be to be resumed.\r\n\r\nAlso reduce the bundle-size default to 20MiB. This is pretty arbitrary, but a smaller default seems better." }, { "commit": "34d649579eb3bd1530aa99f0ed1879e7d3125424", "date": "2022-02-14 13:24:14 -0600", "subject": "Bundle files in the repository during backup.", "body": "Bundle (combine) smaller files during backup to reduce the number of files written to the repository (enable with --bundle). Reducing the number of files is a benefit on all file systems, but especially so on object stores such as S3 that have a high file creation cost. Another benefit is that zero-length files are only stored as metadata in the manifest.\r\n\r\nFiles are batched up to bundle-size and then compressed/encrypted individually and stored sequentially in the bundle. The bundle id and offset of each file is stored in the manifest so files can be retrieved randomly without needing to read the entire bundle. Files are ordered by timestamp descending when being assigned to bundles to reduce the amount of random access that needs to be done. The idea is that bundles with older files can be read in their entirety on restore and only bundles with newer files will get fragmented.\r\n\r\nBundles are a custom format with metadata stored in the manifest. Tar was considered but it is too limited a format, the major issue being that the size of the file must be known in advance and that is very contrary to how pgBackRest works, especially once we introduce page-level incremental backups.\r\n\r\nBundles are stored numbered in the bundle directory. Some files may still end up in pg_data if they are added after the backup is complete. backup_label is an example.\r\n\r\nCurrently, only the backup command works in batches. The restore and verify commands use the offsets to pull individual files out of the bundle. It seems better to finalize how this is going to work before optimizing the other commands. Even as is, this is a major step forward, and all commands function with bundling.\r\n\r\nOne caveat: resume is currently not supported when bundle is enabled." }, { "commit": "8046f0630715015d5adcced570c54e9a183c0e73", "date": "2022-02-14 11:07:02 -0600", "subject": "Do not retry fatal errors.", "body": "There is some evidence that retrying fatal errors, especially out of memory errors, may cause lockups. It makes sense to report fatal errors as quickly as possible and bypass retries. This may or not fix the lockup issue but it is worth doing either way.\r\n\r\nFor now, the only fatal errors will be AssertError and MemoryError." }, { "commit": "8d0cce66f83abc1ff02a985ec3f0d7e285c09d72", "date": "2022-02-13 15:19:31 -0600", "subject": "Use normal error for protocol module error retry test.", "body": "Asserts will not be retried in a future commit, so adjust this test now to use non-assert errors." }, { "commit": "8573a2df14d0bfd5578333b68147cc12c93c6f96", "date": "2022-02-13 15:11:59 -0600", "subject": "Improve protocol module error test for protocolClientFree().", "body": "Using an assert here was never ideal and won't work once we start handling fatal errors differently." }, { "commit": "551e5bc6f67c91cf1e11da0cefe24f9650353adc", "date": "2022-02-11 08:11:39 -0600", "subject": "Retry errors in S3 batch file delete.", "body": "If the entire batch failed it would be retried, but individual file errors were not retried. This could cause pgBackRest to terminate during expiration or when removing an unresumable backup.\r\n\r\nRather than retry the entire batch, delete the errored files individually to take advantage of the HTTP retry rather than adding a new retry loop. These errors seem rare enough that it should not be a performance issue." }, { "commit": "b26097f8d8a7e2af6e9a21576994cbb47110c6f4", "date": "2022-02-09 10:18:39 -0600", "subject": "Add backup type filter to info command.", "body": "Support --type option in the info command to display only a specific backup type." }, { "commit": "cb630ffe3b5c819b5e97ac11d5974000bd44d632", "date": "2022-02-09 09:32:23 -0600", "subject": "Remove logic that tried to determine additional file system compression.", "body": "In theory, the additional stat() call after a file has been copied to the repo can determine if additional compression has been applied by the file system. However, it has been a very long time since we tested this in practice. There are currently no unit tests that accurately test this feature since it requires a compressed file system like ZFS to work, which never seemed worth the extra cost.\r\n\r\nIt can also add a lot of time to backups if there are a large quantity of small files.\r\n\r\nIn addition, it stands as a blocker for combining files for small file support since it is no longer possible to get per-file sizes from the viewpoint of the file system. There are several ways this could be reworked but none of them are easy while at the same time maintaining current info functionality.\r\n\r\nIt doesn't seem worth keeping an untested feature that will only work in some special cases (if it still works) when it is blocking development." }, { "commit": "755bfc4d4015a68fa06890d1d62515b8bbbc3a64", "date": "2022-02-07 08:14:28 -0600", "subject": "Fix invalid file descriptor being passed to close().", "body": "Coverity pointed out that a negative number could be passed to close(), which means the lock file would not get closed until the process ended. Proper execution does not require the file to be closed, but it is better to correctly free resources that are no longer needed." }, { "commit": "5cbb4a3e6b13598b90cae46fad28c31604f9f094", "date": "2022-02-04 15:55:24 -0600", "subject": "Remove unused backup-standby option from stanza commands." }, { "commit": "7dd657b7dd163ded9fbf7caef37eea7e08bc5a37", "date": "2022-02-04 12:48:50 -0600", "subject": "Simplify filename construction in command/verify module.", "body": "Remove some duplicated code by tracking the backup label and constructing the filename only when needed." }, { "commit": "b1da4e84e8b1fda0299931dbcb9b057ce0d3ce64", "date": "2022-02-02 14:39:39 -0600", "subject": "Revert Minio to prior release.", "body": "The most recent release of Minio has broken CI builds but there is no logging to indicate what is wrong.\n\nFor now, just use the prior release to get CI builds working again. This kind if breakage is not uncommon for Minio but they usually resolve it in the next release." }, { "commit": "9b2f10dbb49a8b869cef2c419d9957ed4c550016", "date": "2022-01-31 16:48:28 -0600", "subject": "Refactor lock code.", "body": "Update lock code to use standard common/io functions and module patterns. This module was developed before the common/io module existed and our patterns had stabilized." }, { "commit": "22734eb376b3183924cb7794f9111c1eeb5d92d6", "date": "2022-01-31 10:03:56 -0600", "subject": "Add ioBufferReadNewOpen() and ioBufferWriteNewOpen().", "body": "These are convenience functions to make the code a bit more compact where possible." }, { "commit": "e62e6664d3654dd63ae2eda64f981a6936b822d3", "date": "2022-01-26 13:26:22 -0600", "subject": "Remove release note about the default port being provisional.", "body": "The default port has now been approved by IANA." }, { "commit": "cf5b3a302f0835be993a7f795038d4d676cd6f01", "date": "2022-01-26 13:22:31 -0600", "subject": "Fix language in rh7 test container for aarch64.", "body": "The /etc/profile.d/lang.sh script was causing issues but it does not exist on amd64, so it seems the easiest thing was to remove it.\n\nFix how 32-bit VMs are determined now that another 64-bit architecture has been added.\n\nAnd remove some obsolete VM hashes." }, { "commit": "da0f3a85535df3389c5023afc637124acc6405d3", "date": "2022-01-26 10:15:14 -0600", "subject": "Download correct key on aarch64 in the user guide.", "body": "The default key that gets installed is for amd64. This is only required for RHEL8 it seems, so something is definitely amiss." }, { "commit": "e4df5b7d3846134832b9315e22786f679d63a46b", "date": "2022-01-24 15:21:07 -0600", "subject": "Simplify manifest file defaults.", "body": "Previously manifest load required two passes through the file list, one to load the data and one to set the defaults. This required each file to be packed twice.\r\n\r\nInstead simply note that the file value is default and then set the file defaults when they are loaded from the manifest. This is made possible by the different internal/external representations for files so the same method cannot be applied to paths and links.\r\n\r\nThis change seems to resolve the performance issues noted in 61ce586 but there is no obvious reason why." }, { "commit": "044138fbb19f02ec433e52c42684b1ecc616c624", "date": "2022-01-23 15:07:13 -0500", "subject": "Centralize common PostgreSQL options in the user guide.", "body": "Centralize these options so they are consistent across clusters.\n\nAlso, there were some options that the user doesn't really need to see, .e.g. log_line_prefix. These can be set in advance so they don't need to be part of the documentation." }, { "commit": "13623d6ee50794e87ceac37c61e72ba3a92ad8b6", "date": "2022-01-23 13:53:52 -0500", "subject": "Create create_test_table() function earlier in user guide.", "body": "This function (which creates lots of tables) is generally useful for testing (not just stress testing) so create it as soon as the cluster is created.\n\nAlso add the data parameter which will insert a single row into the table so the file on disk is not zero bytes." }, { "commit": "ca13f11b4a3858587c7055b0b2b4a8b396dd8451", "date": "2022-01-22 17:20:25 -0500", "subject": "Optimization for jsonFromStrInternal().", "body": "This is an extremely hot code path when saving the manifest so every little bit helps." }, { "commit": "61ce58692f3092b72833b6e9ee87037f45b3b702", "date": "2022-01-21 17:05:07 -0500", "subject": "Pack manifest file structs to save memory.", "body": "Manifests with a very large number of files can use a considerable amount of memory. There are a lot of zeroes in the data so it can be stored more efficiently by using base-128 varint encoding for the integers and storing the strings in the same allocation.\r\n\r\nThe downside is that the data needs to be unpacked in order to be used, but in most cases this seems fast enough (about 10% slower than before) except for saving the manifest, which is 10% slower up to 10 million files and then gets about 5x slower by 100 million (two minutes on my M1 Mac). Profiling does not show this slowdown so I wonder if this is related to the change in memory layout. Curiously, the function that increased most was jsonFromStrInternal(), which was not modified. That gives more weight to the idea that there is some kind of memory issue going on here and one hopes that servers would be less affected. Either way, they largest use cases we have seen are for about 6 million files so if we can improve that case I believe we will be better off.\r\n\r\nFurther analysis showed that most of the time was taken up writing the size and timestamp fields, which makes almost no sense. The same amount of time was used if they were hard-coded to 0, which points to some odd memory issue on the M1 architecture.\r\n\r\nThis change has been planned for a while, but the particular impetus at this time is that small file support requires additional fields that would increase manifest memory usage by about 20%, even if the feature is not used.\r\n\r\nNote that the Pack code has been updated to use the new varint encoder, but the decoder remains separate because it needs to fetch one byte at a time." }, { "commit": "575ae77c0dee12b932469d479b2e45dbaf389737", "date": "2022-01-21 15:57:22 -0500", "subject": "Convert varNewUInt64() to VARUINT64() where possible in manifest.", "body": "VARUINT64() does not require memory to be allocated from the mem context so should be used when possible." }, { "commit": "4a73a0286387f3f2185f3f193f9c20ed0d7d926d", "date": "2022-01-21 15:22:48 -0500", "subject": "Simplify manifest defaults.", "body": "Manifest defaults for user, group, and mode were previously generated by scanning the data to find the most common values. This was very accurate but slow and complicated. It could also lead to surprising changes in the manifest when a default value suddenly changed.\r\n\r\nInstead, use the $PGDATA path to generate defaults. In the vast majority of cases the same user/group should own all the path/files and the default file mode is easily derived from the path mode. There may be some edge cases where this generates larger manifests, but in general it reduces time and complexity when saving the manifest.\r\n\r\nRemove the MCV code since it is longer longer used." }, { "commit": "b0db4b8ff00db913cb69308017a56155d6070830", "date": "2022-01-21 08:52:51 -0500", "subject": "Simplify base path mode in mock/all integration tests.", "body": "Change the mode back to 0700 earlier to reduce churn in the expect logs.\n\nThis will be especially important in a future commit that gets the defaults exclusively from the base path." }, { "commit": "8c062e1af824c3dc80ff94b3df8d433f95e88d0f", "date": "2022-01-20 14:01:10 -0500", "subject": "Remove primary flag from manifest.", "body": "This flag was only being used by the backup command after manifestNewBuild() and had no other uses. There was a time when it was important for integration testing but the unit tests now fulfill this role.\r\n\r\nSince backup is the only code concerned with the primary flag, move the code into the backup module.\r\n\r\nWe don't have any cross-version testing but this change was tested manually with the most recent version of pgBackRest to make sure it was tolerant of the missing primary info. When an older version of pgBackRest loads a newer manifest the primary flag will always be set to false, which is fine since it is not used." }, { "commit": "16559d9e421c66596d21b047c8afe1b2c96ccd82", "date": "2022-01-20 08:41:05 -0500", "subject": "Use the PG_FILE_POSTMTRPID constant where appropriate.", "body": "Do the same in Perl with the MANIFEST_FILE_POSTMTRPID constant." }, { "commit": "4ca0590a51a665535fbba903b8f55e10557b606a", "date": "2022-01-20 08:06:42 -0500", "subject": "Refactor backupProcessQueue() to use BackupJobData struct.", "body": "BackupJobData has several members that backupProcessQueue() needs so it is more efficient to use them rather than passing them separately or getting them from the configuration." }, { "commit": "e21ba7c92bc15e9314eefa80cef8f17cf2f0e224", "date": "2022-01-18 17:40:53 -0500", "subject": "Remove extra spaces." }, { "commit": "f84909f85f7dfb202ac812c67665cc590a5acad4", "date": "2022-01-18 17:26:30 -0500", "subject": "Fix issue with Posix read offset handling after an error.", "body": "Coverity pointed out that -1 could be passed to lseek() (added in a79034ae) after a file failed to open because it is missing. Overall it seems simpler to enclose the success code in an else block to prevent any repeats of this mistake in the future.\n\nThis was not an active bug because there are currently no cases where we do read offsets in a file that is allowed to be missing.\n\nAlso remove the result flag since it is easier to just check that the file descriptor is valid." }, { "commit": "91721e391a4d83a405a0d1be31b854312f9f157d", "date": "2022-01-17 15:39:31 -0500", "subject": "Fix typo." }, { "commit": "b791f1c82f53fa16866837d4544b58243c682f7f", "date": "2022-01-15 14:33:38 -0500", "subject": "Implement restore ownership without updating manifest internals.", "body": "Updating the manifest this way was not a great idea because it broke abstraction for the object. This meant certain changes to the interface and internals were not possible because the code was modifying internal manifest data.\n\nInstead track the user replacements entirely in the restore module.\n\nThis also has the benefit of eliminating a pass over the manifest path/file/link lists." }, { "commit": "3097acd73a21b630aa5352be57a82b031930f96b", "date": "2022-01-13 08:46:14 -0500", "subject": "Add support for AWS S3 server-side encryption using KMS.", "body": "AWS S3 integrates with AWS Key Management Service (AWS KMS) to provide server side encryption of S3 objects. This integration protects objects under encryption keys that never leave AWS KMS unencrypted." }, { "commit": "92ea3e05fb43e5272a19fb1e3208412b75008c24", "date": "2022-01-11 19:15:09 -0500", "subject": "cfgOptionGroupName() replacement missed in aced5d47." }, { "commit": "a79034ae2f6a8ec66b84c62ab6cd4042d2c1a249", "date": "2022-01-11 14:42:53 -0500", "subject": "Add read range to all storage drivers.", "body": "The range feature allows reading out an arbitrary chunk of a file and will be important for efficient small file support.\r\n\r\nNow that all drivers are required to support ranges remove the storageFeatureLimitRead feature flag that was implemented only by the Posix driver." }, { "commit": "2fd100bf12760a4bb1996fb464ae54b5c379988e", "date": "2022-01-10 17:40:18 -0500", "subject": "Rename ConfigOptionGroupData.indexDisplay to indexName.", "body": "This is more consistent with the function name and purpose." }, { "commit": "2cddbbdee01391ef0885f9262f3acc488382eee2", "date": "2022-01-10 17:20:48 -0500", "subject": "Remove obsolete cfgOptionHostPort()/cfgOptionIdxHostPort().", "body": "These functions were made obsolete by the refactor in 6a124584." }, { "commit": "7c627c12b735345cc3d733ae66a52b1fe0030e47", "date": "2022-01-10 17:00:58 -0500", "subject": "Refactor option and option group config structs.", "body": "This allows individual structs to be assigned to variables to make the code easier to read and perhaps a bit more efficient." }, { "commit": "aeecb500f54561f7d52fa1b637f7f6e1f7a668de", "date": "2022-01-10 14:47:29 -0500", "subject": "Improve implementation of cfgOptionIdxName().", "body": "Cache option names after they are generated rather than regenerating them each time." }, { "commit": "aced5d47ed8e57ca7f36f92fe068f80b66f81649", "date": "2022-01-10 09:10:06 -0500", "subject": "Replace cfgOptionGroupIdxToKey() with cfgOptionGroupName().", "body": "Do the replacement anywhere cfgOptionGroupIdxToKey() is being used to construct a group name in a message. cfgOptionGroupName() is better for this case since it also includes the name of the group so that it does not need to be repeated in each message." }, { "commit": "9657f1b32519272edaaefd379355137afd554e82", "date": "2022-01-09 19:53:05 -0500", "subject": "Dynamically allocate index to key index map.", "body": "Now the config/config module has no notion of max option indexes. The config/parse still needs the max but this will be removed at a later date." }, { "commit": "e4b48eb430a564184ab739908df55813dd970aa4", "date": "2022-01-09 19:43:44 -0500", "subject": "Fix inconsistent group display names in messages.", "body": "In other instances there are no dashes, e.g. repo1 or pg1. Make these messages match." }, { "commit": "5f78a5fc18120c67cb8a68f83a4fb9bcf96982e2", "date": "2022-01-09 13:19:43 -0500", "subject": "Add ioCopy().", "body": "Functionality to copy from IoRead to IoWrite is frequently used so centralize it. This also simplifies coverage testing in places where a loop was required before." }, { "commit": "47954774c6cc70a6fc9a6c8836d82a87e341ea11", "date": "2022-01-09 10:11:00 -0500", "subject": "Combine encrypted backupFile() tests with unencrypted tests.", "body": "This makes it easier to comment out all the tests while developing without getting unused variable errors." }, { "commit": "d866dd5c297dc046581134a108032cf9694cae0e", "date": "2022-01-07 14:09:58 -0500", "subject": "Add backup LSNs to info command output.", "body": "The backup LSNs are useful for performing LSN-based PITR. LSNs will not be displayed in the general text output (without --set) because they are probably not useful enough to deserve their own line." }, { "commit": "bb4b30ddd3f80c246c565eb4769efc82b9509648", "date": "2022-01-06 15:34:04 -0500", "subject": "Remove support for PostgreSQL 8.3/8.4.", "body": "There is no evidence that users need 8.3/8.4 anymore but it does cost us in terms of development and testing, especially now that we have a number of new backup/restore features planned.\r\n\r\nIt seems to make sense to remove this support now. If there are users who need to use/migrate from these versions they can use an older version of pgBackRest." }, { "commit": "ef62ef23799f081582295f1f3e8a92691b628f53", "date": "2022-01-06 14:24:32 -0500", "subject": "Fix comment missed in 6bd280f7." }, { "commit": "fdbeb8e7d64754d35dd30e4f871834756c5d77ca", "date": "2022-01-06 14:22:56 -0500", "subject": "Fix typo in error message." }, { "commit": "a82f0179cdd5742e5c5b1e0bdfd23166753fdb38", "date": "2022-01-04 16:11:27 -0500", "subject": "Note that replications slots are not restored.", "body": "Update documentation and help to note that replication slots are not restored and reference the PostgreSQL documentation to explain why." }, { "commit": "226cfbdcde55ca384e74d7d1348c447eecd079fa", "date": "2022-01-04 15:48:00 -0500", "subject": "Fix typo." }, { "commit": "591a72c59ec403341d29eaafe3030f4651732192", "date": "2022-01-03 08:57:05 -0500", "subject": "Update config.guess and config.sub to latest versions." }, { "commit": "75e4580919ed139f00db7c4c3141dbefc3da131b", "date": "2022-01-03 08:52:25 -0500", "subject": "Begin v2.38 development." }, { "commit": "f18f2d9991da29533ad8e9859074c4f39185420f", "date": "2022-01-03 08:43:55 -0500", "subject": "v2.37: TLS Server" }, { "commit": "62fbee72ad319f92c9410ac8dbab2f81fe945a36", "date": "2022-01-01 10:50:16 -0500", "subject": "Update LICENSE.txt and PostgreSQL copyright for 2022." }, { "commit": "d6ebf6e2d67331a838f53beda1c186c527b56a8e", "date": "2021-12-30 18:54:36 -0500", "subject": "Remove dead test code." }, { "commit": "fccb7f7dd45c6c373d0cfa74b90d69ca483aa3af", "date": "2021-12-28 17:39:22 -0500", "subject": "Add release note regarding IANA approval of the default TLS port." }, { "commit": "6a12458440168f13cb05d70f36ea54b1860e390c", "date": "2021-12-16 10:30:59 -0500", "subject": "Parse protocol/port in S3/Azure endpoints.", "body": "Utilize httpUrlNewParseP() to parse endpoint and port from the URL in the S3 and Azure helpers to avoid issues where protocol was not expected to be part of the URL." }, { "commit": "f06101de77a980c7e4115762f2fc301280aa4127", "date": "2021-12-16 09:47:04 -0500", "subject": "Add TLS server documentation.", "body": "Add documentation and make the feature visible." }, { "commit": "615bdff4030a31bfedfe7df04676e3948ec9c2c0", "date": "2021-12-14 14:53:41 -0500", "subject": "Fix socket leak on connection retries.", "body": "This leak was caused by the file descriptor variable getting clobbered after a long jump. Mark it as volatile to fix.\r\n\r\nTesting this is a bit complex because the issue only happens in optimized builds, if at all. Put the test into the performance suite, which is always optimized, until a better idea presents itself." }, { "commit": "a73fe4eb966f9685f6e4179c397a10c1e7f15f19", "date": "2021-12-10 15:53:40 -0500", "subject": "Fix restore delta link mapping when path/file already exists.", "body": "If a path/file was remapped to a link using either --link-map or --link-all there would be no affect if the path/file already existed. If a link existed it would be properly updated and converting a link to a path/file also worked.\r\n\r\nThe issue happened during delta cleanup, which failed to check if the existing path/file had been remapped to a link.\r\n\r\nAdd checks for newly mapped path/file links and remove the old path/file we required." }, { "commit": "19a7ec69debfe6587fcc1163451896590c96bf21", "date": "2021-12-10 15:04:55 -0500", "subject": "Close expect log file when unit test completes.", "body": "This did not cause any issues, but it is better to explicitly close open files." }, { "commit": "c38e2d31709804eb4b9125a15ad84c8fc813f366", "date": "2021-12-08 15:00:19 -0500", "subject": "Add verb to HTTP error output.", "body": "This makes it easier to debug HTTP errors." }, { "commit": "be4ac3923cb77873da298a30aca5d847b3c635af", "date": "2021-12-08 13:57:26 -0500", "subject": "Error when restore is unable to find a backup to match the time target.", "body": "This was previously a warning but the warning is easy to miss so a lot of time may be lost restoring and recovering a backup that will not hit the target.\r\n\r\nSince this is technically a breaking change, add an \"important note\" about the change to the release." }, { "commit": "672330593789f07aaad90bbafcd2597cbc602686", "date": "2021-12-08 12:29:20 -0500", "subject": "Add warning when checkpoint_timeout exceeds db-timeout.", "body": "In the backup command, add a warning if start-fast is disabled and the PostgreSQL checkpoint_timeout is greater than db-timeout.\r\n\r\nIn such cases, we might timeout before the checkpoint occurs and the backup really starts." }, { "commit": "bd2ba802db11c505ec69943fa81b2b379073fbf4", "date": "2021-12-08 10:16:41 -0500", "subject": "Check that clusters are alive and correctly configured during a backup.", "body": "Fail the backup if a cluster stops or the standby is promoted. Previously, shutting down the primary would cause an error but it was not detected until the end of the backup. Now the error will happen sooner and a promotion on the standby will also cause an error." }, { "commit": "7b3ea883c7c010aafbeb14d150d073a113b703e4", "date": "2021-12-07 18:18:43 -0500", "subject": "Add SIGTERM and SIGHUP handling to TLS server.", "body": "SIGHUP allows the configuration to be reloaded. Note that the configuration will not be updated in child processes that have already started.\r\n\r\nSIGTERM terminates the server process gracefully and sends SIGTERM to all child processes. This also gives the tests an easy way to stop the server." }, { "commit": "49145d72bac16498cdbf5eeb3cd6128ea0be0667", "date": "2021-12-07 09:21:07 -0500", "subject": "Add timeline and checkpoint checks to backup.", "body": "Add the following checks:\r\n\r\n* Checkpoint is updated in pg_control after pg_start_backup(). This helps ensure that PostgreSQL and pgBackRest have a consistent view of the storage and that PGDATA paths match.\r\n* Timeline of backup start WAL file matches pg_control. Hard to see how this one could get hit, but we have the power...\r\n* Standby is on the same timeline as the primary. If not, this standby is not following the primary.\r\n* Last standby checkpoint is not greater than the backup checkpoint. If so, this standby is not following the primary.\r\n\r\nThis also requires some additional plumbing to read/write timeline/checkpoint from pg_control and parse timelines from WAL filenames. There were some changes in the backup tests caused by the fact that pg_control now has different contents for each backup.\r\n\r\nThe check to ensure that the required checkpoint was reached on the standby should also be updated to use pg_control (it currently uses pg_control_checkpoint()), but that requires non-trivial changes to the test harness and will need to wait." }, { "commit": "9c76056dd0d1d2b07a89646b087c5c8d36ab97f5", "date": "2021-11-30 16:21:15 -0500", "subject": "Add error type and message to CHECK() macro.", "body": "A CHECK() worked exactly like ASSERT() except that it was compiled into production code. However, over time many checks have been added that should not throw AssertError, which should be reserved for probable coding errors.\n\nAllow the error code to be specified so other error types can be thrown. Also add a human-readable message since many of these could be seen by users even when there is no coding error.\n\nUpdate coverage exceptions for CHECK() to match ASSERT() since all conditions will never be covered." }, { "commit": "0895cfcdf7d3f15b8029f73ed62c6094d30724b3", "date": "2021-11-30 13:23:11 -0500", "subject": "Add HRN_PG_CONTROL_PUT() and HRN_PG_CONTROL_TIME().", "body": "These macros simplify management of pg_control test files.\n\nCentralize time updates for pg_control in the command/backup module. This caused some time updates in the logs.\n\nFinally, move the postgres module after the storage module so it can use storage macros." }, { "commit": "01ac6b6cac86ea857e54a3b1c45077df1e128a75", "date": "2021-11-30 08:28:36 -0500", "subject": "Autogenerate test system identifiers.", "body": "hrnPgControlToBuffer() and hrnPgWalToBuffer() now generate the system id based on the version of Postgres. If a value less than 100 is specified for systemId then it will be added to the default system id so there can be multiple ids for a single version of PostgreSQL.\n\nAdd constants to represent version system ids in tests. These will eventually be auto-generated.\n\nThis changes some checksums and we no longer have big-endian tests systems, so X those checksums out so it is obvious they are no longer valid." }, { "commit": "3f7409019df112ec50efb6c3db6f7780c9a63c87", "date": "2021-11-24 16:09:45 -0500", "subject": "Ensure ASSERT() macro is always available in test modules.", "body": "Tests that run without DEBUG for performance did not have ASSERT() and were using CHECK() instead.\n\nInstead ensure that the ASSERT() macro is always available in tests." }, { "commit": "dcb4f09d8315e92c0877b589f3fa9b7f0fa65f93", "date": "2021-11-23 09:37:12 -0500", "subject": "Revert changes to backupFilePut() made in 1e77fc3d.", "body": "These changes were made obsolete by a3d7a23a." }, { "commit": "7e35245dc3416238a84a43abbecdf976170dea91", "date": "2021-11-23 08:07:31 -0500", "subject": "Use ASSERT() or TEST_RESULT*() instead of CHECK() in test modules." }, { "commit": "a3d7a23a9d90611a3d31947598fbea240b250710", "date": "2021-11-22 12:52:37 -0500", "subject": "Use infoBackupDataByLabel() to log backup size.", "body": "Eliminate summing and passing of copied files sizes for logging backup size.\r\n\r\nInstead, utilize infoBackupDataByLabel() to pull the backup size for the log message." }, { "commit": "1a0560d363d28737befb8c222647783d4fc2ca29", "date": "2021-11-19 12:22:09 -0500", "subject": "Allow y/n arguments for boolean command-line options.", "body": "This allows boolean boolean command-line options to work like their config file equivalents.\r\n\r\nAt least for now this behavior will remain undocumented since all examples in the documentation will continue to use the standard syntax. The idea is that it will \"just work\" when options are copied out of config files rather than generating an error." }, { "commit": "2d963ce9471808172f879916c3f3accc35f14d56", "date": "2021-11-18 17:23:11 -0500", "subject": "Rename server-start command to server." }, { "commit": "1f14f45dfb0d1677a695719381cbd5a8a3c6c986", "date": "2021-11-18 16:18:10 -0500", "subject": "Check archive immediately after backup start.", "body": "Previously the archive was only checked at the end of the backup to ensure all WAL required to make the backup consistent was present. The problem was that if archiving was not functioning then the backup had to complete before the user found out, which could be a while if the database was large enough.\r\n\r\nAdd an archive check immediately after backup start so failures are reported earlier.\r\n\r\nThe trick is to determine which WAL to check. If the repo is new there may not be any WAL in it and pg_start_backup() will not switch the WAL segment if it is empty. These are both likely scenarios when setting up and/or testing pgBackRest.\r\n\r\nIf the WAL segment is switched by pg_start_backup(), then check the archive for the segment that was detected prior to backup start. This should be common on normal running clusters with regular activity. Note that this might not be the segment immediately prior to the backup start segment if WAL volume is high.\r\n\r\nIf pg_start_backup() did not switch the WAL then we can force a switch on PostgreSQL >= 9.3 by creating a restore point. In that case the WAL to check will be the backup start WAL. This is most likely to happen on idle systems, during testing, or immediately after a repo switch.\r\n\r\nAn advantage of this approach other than earlier notification is that the backup directory will not be created so no resume will be attempted on the next backup.\r\n\r\nNote that some additional churn was created in backup.c because the load of archive.info needs to be done earlier." }, { "commit": "dea752477ab8e812cdbd717eb2091baf3f5d0906", "date": "2021-11-17 16:39:04 -0500", "subject": "Remove obsolete statement about future multi-repository support." }, { "commit": "0949b4d35fdd04c55927eb6a107d881376dbe73c", "date": "2021-11-16 18:26:21 -0500", "subject": "Add linefeed and remove space." }, { "commit": "809f0bbc638cdd95540e2257383147919f82e8f9", "date": "2021-11-16 11:34:53 -0500", "subject": "Add infoBackupLabelExists().", "body": "This is easier to read than using infoBackupDataByLabel() != NULL.\n\nIt also allows an assertion to be added to infoBackupDataByLabel() to ensure that a NULL return value is not used unsafely." }, { "commit": "1e77fc3d75490b7a1b6a0b31be9298c995ec672f", "date": "2021-11-16 10:21:32 -0500", "subject": "Include backup_label and tablespace_map file sizes in log output.", "body": "In cases where they are returned by postgres, include backup_label and tablespace_map file sizes in the backup size value output in the log." }, { "commit": "6b5322cdad7163d91b43d37d9d8eeaa39ac7f214", "date": "2021-11-16 09:27:15 -0500", "subject": "Add findutils package to RHEL 8 documentation container.", "body": "This package was dropped from the most recent Rocky Linux 8 image." }, { "commit": "df89eff429e9b8fbc68d9e9895badf9719fd31d2", "date": "2021-11-15 16:53:41 -0500", "subject": "Fix typos and improve documentation for the tablespace-map-all option." }, { "commit": "fcae9d35038d454c674921c65beb02b195981480", "date": "2021-11-15 16:42:46 -0500", "subject": "Fix parameter test logging in parseOptionIdxValue()." }, { "commit": "b3a5f7a8e27768c445458e47dad626609814fbb7", "date": "2021-11-15 14:32:22 -0500", "subject": "Add tablespace_map file to command/backup test module.", "body": "The code worked fine but better to have explicit tests for this file." }, { "commit": "e62ba8e85eaf469052960c4fd71ffaf26c1a1baa", "date": "2021-11-12 17:15:45 -0500", "subject": "Add path to pgbench used for stress test in user guide.", "body": "This allows the stress test to run on RHEL." }, { "commit": "43cfa9cef776360e592882c0b787704dbeb36cb3", "date": "2021-11-10 12:14:41 -0500", "subject": "Revive archive performance test.", "body": "This test was lost due to a syntax issue in a58635ac.\n\nUpdate the test to use system() to better mimic what postgres does and add logging so pgBackRest timing can be determined." }, { "commit": "dd96c29f963609fad38dac3349d7fa41e40722bb", "date": "2021-11-10 07:53:46 -0500", "subject": "Refactor postgres/client module with inline getters/setters.", "body": "Extend the pattern introduced in 79a2d02 to the postgres/client module." }, { "commit": "afe77e76e0adf948138d797e227a6f4c7d47c2eb", "date": "2021-11-10 07:31:02 -0500", "subject": "Update contributor for 6e635764." }, { "commit": "6e635764a66278d5a8c2b4d30b23063bc3923067", "date": "2021-11-09 13:24:56 -0500", "subject": "Match backup log size with size reported by info command.", "body": "Properly log the size of files copied during the backup, matching the backup size returned from the info command.\r\n\r\nIn the reference issue, the incremental backup after switchover logs the size of all files evaluated rather than only the size of the files copied in the backup." }, { "commit": "d05d6b87142347cb4891304833db389dcf7f9a81", "date": "2021-11-08 09:39:58 -0500", "subject": "Do not delete manifests individually during stanza delete.", "body": "This appears to have been an attempt to not delete files that we don't recognize, but it only works in narrow cases and could leave the user is a position of not being able to complete the stanza delete without manual intervention. It seems better just to proceed with the delete, especially since the info files have already been removed.\n\nIn addition, deleting the manifests individually could be slow on object stores if there were a very large number of backups." }, { "commit": "bb03b3f41942d0b781931092a76877ad309001ef", "date": "2021-11-04 09:44:31 -0400", "subject": "Refactor switch statements in strIdBitFromZN().", "body": "Coverity does not like fall-throughs either to or from the default case so refactor to avoid that." }, { "commit": "676b9d95dd2467d4bddd402b5cd2b4f445c71944", "date": "2021-11-04 08:19:18 -0400", "subject": "Optional parameters for tlsClientNew().", "body": "There are a number of optional parameters with the same type so this makes them easier to track and reduces churn when new ones are added." }, { "commit": "038abaa71d816cc87b382bd81d3df62ddec9455a", "date": "2021-11-03 15:23:08 -0400", "subject": "Display size option default and allowed values with appropriate units.", "body": "Size option default and allowed values were displayed in bytes, which was confusing for the user.\r\n\r\nThis also lays the groundwork for adding units to time options.\r\n\r\nMove option parsing functions into a common module so they can be used from the build module." }, { "commit": "1b93a772369bbb3a936099e0d9d5cc79bad1e0f6", "date": "2021-11-03 12:14:17 -0400", "subject": "Use void * instead of List * to avoid Coverity false positives.", "body": "Coverity complains that this should be \"List\" but that is clearly not correct." }, { "commit": "2a576477b316238473525e56bc8fc8ea5790455f", "date": "2021-11-03 11:36:34 -0400", "subject": "Add --cmd option.", "body": "Allows users to provide an executable to be used when pgbackrest generates command strings that expect to invoke pgbackrest. These generated commands are written to files by pgbackrest, e.g. recovery.conf." }, { "commit": "c5b5b5880619d0994ab4a8feb3f60ab52170b61b", "date": "2021-11-03 10:36:31 -0400", "subject": "Simplify error handler.", "body": "The error handler used a loop to process try, catch, and finally blocks. This worked fine but static analysis tools like Coverity did not understand that the finally block would always run and so there were false positives about double-free, unfreed resource, etc.\r\n\r\nThis implementation removes the loop, which simplifies everything, and makes it clear that the finally block will always run. This cuts down on Coverity false positives.\r\n\r\nThis implementation also catches lack of coverage on empty catch blocks so a few test fixes were committed separately in d74fe7a.\r\n\r\nA small refactor in backup.c is required because gcc 10.3.1 on Fedora 33 complains that the reason variable may be used uninitialized. It's not clear why this is the case, but reducing the scope of the TRY block fixes the issue." }, { "commit": "cff961ede7e41fa8035ffe7451a22eb5ea0e46c1", "date": "2021-11-03 07:38:06 -0400", "subject": "Centralize logic to build value lists during config rendering.", "body": "This reduces duplication and makes it easier to add new types." }, { "commit": "7f6c513be925c77bc6a177408efcf79f624ffc94", "date": "2021-11-03 07:27:26 -0400", "subject": "Add StringId as an option type.", "body": "Rather the converting String to StringIds at runtime, store defaults in StringId format in parse.auto.c and convert user input to StringId during parsing." }, { "commit": "b13844086d419dc3070bcce4e918b2353bf4887c", "date": "2021-11-01 17:35:19 -0400", "subject": "Use cfgOptionStrId() instead of cfgOptionStr() where appropriate.", "body": "The compress-type, repo-type and log-level-* options have allow lists, which means it is more efficient to treat them as StringIds.\r\n\r\nFor compress-type and log-level-* also update the functions that convert them to enums." }, { "commit": "b237d0cd592bbc6c6ee9280fb7aed264bf79eb9d", "date": "2021-11-01 10:43:08 -0400", "subject": "Remove placeholder bz2 helper data.", "body": "This placeholder data should have been removed when bz2 support was added in a021c9fe053." }, { "commit": "f4e281399a81835821547ea5c78ed7a189914d3d", "date": "2021-11-01 10:27:57 -0400", "subject": "Remove unused protocol log level.", "body": "This log level was used in the Perl code but was never ported to C." }, { "commit": "bc352fa6a8cff7cc08b6c7f3cdfac664d2b0805f", "date": "2021-11-01 10:08:56 -0400", "subject": "Simplify strIdFrom*() functions.", "body": "The strIdFrom*() forced the caller to pick an encoding, which led to a number of TRY...CATCH blocks in the code. In practice the caller does not care which encoding is used as long as the string is valid for some encoding.\r\n\r\nUpdate the strIdFrom*() function to try all possible encodings and only throw an error when the string is not valid for any of them." }, { "commit": "a92d7938197d1035e362390ce467ae827cbae051", "date": "2021-11-01 09:11:43 -0400", "subject": "Update automake version.", "body": "There were no changes to install.sh in this version." }, { "commit": "904b897f5e89542784af64b364a49205e7a6e040", "date": "2021-11-01 09:03:42 -0400", "subject": "Begin v2.37 development." }, { "commit": "42fd6ce4e09ee92614cfbfb6766d9c3a6ba9cc1a", "date": "2021-11-01 08:59:14 -0400", "subject": "v2.36: Minor Bug Fixes and Improvements" }, { "commit": "6abb06248c2829f2c27a7a553d373b0fdf70cfc3", "date": "2021-10-29 11:45:50 -0400", "subject": "Make analytics optional for HTML documentation.", "body": "Analytics should only be added to the current HTML documentation on the website, so exclude them by default." }, { "commit": "13366573261bf2562bc93ef77014f7d16b815e5b", "date": "2021-10-29 10:35:56 -0400", "subject": "Restore some linefeed rendering behavior from before def7d513.", "body": "The new rendering behavior is correct in normal cases, but for the pre-rendered HTML blocks in the command and configuration references it causes a lot of churn. This would be OK if the new HTML was diff-able, but it is not.\n\nGo back to the old behavior of using br tags for this case to reduce churn until a more permanent solution is found." }, { "commit": "c32e000ab92e9e9e5495ddec7c4e347c35801570", "date": "2021-10-28 15:15:49 -0400", "subject": "Use Rocky Linux for documentation builds instead of CentOS.", "body": "Since CentOS 8 will be EOL at the end of the year it makes sense to do this now. The centos:8 image is still used in documentation.xml because changes there require manual testing, which will need to be done at a later date. The changes are not user-facing, however, and can be done at any time.\n\nAlso update CentOS references to RHEL since that is what we are emulating for testing purposes." }, { "commit": "30c589ace7a459f3b3d09b702e314efd412e71d6", "date": "2021-10-28 13:28:49 -0400", "subject": "Fix typo in contributing guide.", "body": "Not sure how this got broken but it was probably an errant search and replace." }, { "commit": "2f1a2877373c7be68d553c7f781299edaf8ff196", "date": "2021-10-28 11:49:00 -0400", "subject": "Add missing assert." }, { "commit": "adc09ffc3bccb24c83a471c8af1f9bf68f2cf9c8", "date": "2021-10-28 08:10:43 -0400", "subject": "Minor fix for lower-casing of option summaries.", "body": "This works with existing cases and fixes \"I/O\"." }, { "commit": "fa564ee1969229b5cf60d2479d8ace85325f4db3", "date": "2021-10-27 11:08:32 -0400", "subject": "Improve documentation for cmd-ssh, repo-host-cmd, pg-host-cmd options.", "body": "Use \"command\" instead of \"exe\" and make the descriptions more consistent." }, { "commit": "e1f6c066b3da11fd21b1155c90370c3fa2da06b7", "date": "2021-10-27 10:52:39 -0400", "subject": "Improve documentation for buffer-size option." }, { "commit": "1f7c7b7dda1c736fab2673084498fc7c220b742a", "date": "2021-10-26 16:56:44 -0400", "subject": "Fix test descriptions in common/typeVariantTest." }, { "commit": "d74fe7a222c1e1ae0f02addbeb712f8946d3d731", "date": "2021-10-26 13:53:44 -0400", "subject": "Add coverage for empty CATCH() blocks.", "body": "Currently empty CATCH() blocks are always marked as covered because of the loop structure of error handling.\n\nA prototype implementation of error handling without looping has shown that these CATCH() blocks are not covered without new tests. Whether or not that prototype gets committed it is worth adding the tests." }, { "commit": "e2eea974c144f77448aa9d5fbb55c933b70ea5ad", "date": "2021-10-26 12:09:41 -0400", "subject": "Add assertion for Coverity.", "body": "Coverity thinks this value might be NULL but that should not be possible because of the TRY...CATCH block." }, { "commit": "4f10441574761c9cd4e31cdef750742e004ae669", "date": "2021-10-26 08:25:21 -0400", "subject": "Add missing paragraph tags in coding standards." }, { "commit": "7fb99c59c88fe11c679d6ba7835f995a969462c0", "date": "2021-10-26 07:46:48 -0400", "subject": "Use externed instead of extern'd in comments.", "body": "This is mostly to revert some comment changes in b11ab9f7 that will break the ppc64le patch, but at the same time keep the spelling consistent in all comments and documentation.\n\nAlso revert some space changes for the same reason." }, { "commit": "653ffcf8d98ebfe94ae44ed54b4a295428c57850", "date": "2021-10-25 15:42:28 -0400", "subject": "Adjustments for new breaking change in Azurite.", "body": "Azurite released another breaking change (see fbd018cd, 096829b3, c38d6926, and Azurite issue 1039) so make adjustments as needed to documentation and tests.\n\nAlso remove some dead code that hid the repo-storage-host option and was made obsolete by all these changes." }, { "commit": "13d4559708819787ad05be6f37ec0badb0eccae5", "date": "2021-10-25 15:31:39 -0400", "subject": "Check return value of getsockopt().", "body": "Checking the return value is not terribly important here, but if setsockopt() fails it is likely that bind() will fail as well. May as well get it over with and this makes Coverity happy." }, { "commit": "1152f7a7d64e69eed1d9e74b48a308f6c742c28a", "date": "2021-10-25 12:56:33 -0400", "subject": "Fix mismatched parameters in tlsClientNew() call.", "body": "3879bc69 added this call and the parameters were not quite right but in way that the compiler decided they were OK. It was mostly working but TLS verification was disabled if caPath was NULL, which is not OK." }, { "commit": "a1a2284c881ba6c3b9b1c316b31e0583c006f1af", "date": "2021-10-25 09:01:22 -0400", "subject": "Fix typos in error messages." }, { "commit": "3879bc69b888daa04d2ca98a2d1219cf22519ddc", "date": "2021-10-22 18:31:55 -0400", "subject": "Add WebIdentity authentication for AWS S3.", "body": "This allows credentials to be automatically acquired in an EKS environment." }, { "commit": "51785739f44b624091246c48af6defe97c30d7a7", "date": "2021-10-22 18:02:20 -0400", "subject": "Store config values as a union instead of a variant.", "body": "The variants were needed to easily serialize configurations for the Perl code.\r\n\r\nUnions are more efficient and will allow us to add new types that are not supported by variants, e.g. StringId." }, { "commit": "2cea005f740d640290a9948595f5933833e30e7d", "date": "2021-10-22 17:19:16 -0400", "subject": "Fix segfault on invalid GCS key file." }, { "commit": "cb36fec102855bf268ec5234bbb5261be98bdc61", "date": "2021-10-21 17:48:00 -0400", "subject": "Add analytics to the HTML documentation." }, { "commit": "a63e732987bc1f6f26514568dabc6c0b23df07ab", "date": "2021-10-21 17:25:32 -0400", "subject": "Fix indentation." }, { "commit": "78e1bd333068c4a857054490698115fa2c698e0b", "date": "2021-10-21 17:10:00 -0400", "subject": "Move v1 documentation links out of the introduction.", "body": "There should be few if any users running v1 now so these links do not need to be so prominent." }, { "commit": "861df2a73cafbd49049dccdc55d5214b00dd3cec", "date": "2021-10-21 17:02:46 -0400", "subject": "Add GitHub repository link to index.html and README.md." }, { "commit": "1cb8ae15de5b4276682bdd9825ca97012cd43855", "date": "2021-10-21 13:51:59 -0400", "subject": "Fix incorrect host name in user guide.", "body": "The text indicates to populate the pg-primary IP address into the pg_hba.conf file to allow replication connections. It should indicate to populate the pg-standby IP address" }, { "commit": "b11ab9f799aa6fc32dd03e96e8a0428d5c83d9ae", "date": "2021-10-21 13:31:22 -0400", "subject": "Fix typos." }, { "commit": "8ad6b7330e1ee6bcbc0f06ec0562a433e7888f44", "date": "2021-10-21 09:20:40 -0400", "subject": "Fix outdated comment.", "body": "This check was moved from within the path checks at some point but the comment did not get updated." }, { "commit": "fbd018cd56482efff425beb4026fe22482115138", "date": "2021-10-20 08:22:37 -0400", "subject": "Allow S3/Azure Docker images to be specified in user guide.", "body": "It is not uncommon for the S3/Azure emulators we use to introduce breaking changes without warning. If that happens the documentation can still be built by specifying a working version of the image. In general, it is better to let the version float so we know when things break.\n\nAzurite has yet another breaking change coming up (see 096829b3, c38d6926, and Azurite issue 1039) so set azure-image at the current version until the breaking change has been released." }, { "commit": "5dfdd6dd5b7b43dc3a223b9552aed4052d0db3aa", "date": "2021-10-19 12:45:20 -0400", "subject": "Add -Werror -Wfatal-errors -g flags to configure --enable-test.", "body": "These flags are used for all tests but it was not possible to add them to configure before the change in 046d6643. This is especially important for adhoc tests to ensure the flags are not forgotten.\n\nRemove the flags from test make commands where they were being applied.\n\nThere is no change for production builds." }, { "commit": "046d6643373859c5e848a97e06389ed2aa553723", "date": "2021-10-19 12:14:09 -0400", "subject": "Set most compiler flags as late as possible in configure.", "body": "Some flags, e.g. -Wfatal-errors, will cause tests in configure to behave incorrectly so we have not been able to add them to --enable-test.\n\nAdd the compiler flags as late as possible so configure checks are not affected. This will allow us to add flags that we need for testing without having to explicitly pass them to make." }, { "commit": "e443e3c6c05c9d65a67dac0c8430b59239fbc1b8", "date": "2021-10-19 09:06:06 -0400", "subject": "Add br tags for HTML documentation rendering missed in def7d513." }, { "commit": "4c2d89eb66e11017e2e73ad4171e4493c28acdad", "date": "2021-10-18 16:43:19 -0400", "subject": "Fix typos." }, { "commit": "6cc8e45df68c299990c0ad1f40c53b9282cb46db", "date": "2021-10-18 14:45:36 -0400", "subject": "Add missing paragraph tag in user guide." }, { "commit": "ccc255d3e05d8ce2b6ac251d1498f71b04098a86", "date": "2021-10-18 14:32:41 -0400", "subject": "Add TLS Server.", "body": "The TLS server is an alternative to using SSH for protocol connections to remote hosts.\n\nThis command is currently experimental and intended only for trial and testing. As such, the new commands and options will not show up in the command-line help unless directly requested." }, { "commit": "09fb9393f14b47effebaecc449a97ad07ef4c752", "date": "2021-10-18 14:02:05 -0400", "subject": "Write command configuration overrides explicitly.", "body": "If not written explicitly then it is impossible to distinguish the override from a NULL, which indicates no override." }, { "commit": "90f7f11a9f71152185219bbb57bf1de001e3a91b", "date": "2021-10-18 12:22:48 -0400", "subject": "Add missing static keywords in test modules." }, { "commit": "4570c7e27528400373ece8dc7bd348baf3ff064e", "date": "2021-10-18 11:32:53 -0400", "subject": "Allow error buffer to be resized for testing.", "body": "Some tests can generate very large error messages for diffs and they often get cut off before the end.\n\nAlso fix a test so it does not create too large a buffer on the stack." }, { "commit": "838ee3bd08c739e3dcf611e9bddfaa6c8acbb2aa", "date": "2021-10-18 11:05:53 -0400", "subject": "Increase some storage test timeouts.", "body": "32-bit Debian 9 is sometimes timing out on these tests so increase the timeouts to make the tests more reliable." }, { "commit": "6b9e19d423d99d3063c4bff3d3533b1e5081e4cb", "date": "2021-10-16 12:35:47 -0400", "subject": "Convert configuration optional rules to pack format.", "body": "The previous format was custom for configuration parsing and was not as expressive as the pack format. An immediate benefit is that commands with the same optional rules are merged.\n\nDefaults are now represented correctly (not multiplied), which simplifies the option default functions used by help." }, { "commit": "360cff94e4e9e1ab5a690a1f5c38eb278158a892", "date": "2021-10-16 12:33:31 -0400", "subject": "Update 32-bit test container to Debian 9.", "body": "Also rebalance PostgreSQL version integration tests." }, { "commit": "0e84c19a9fde0480b30078f5d3b419267b2f7673", "date": "2021-10-15 17:50:54 -0400", "subject": "Remove allow range from pg-socket-path option.", "body": "The allow range was never processed because the string type does not allow ranges, but it is wasteful to have it in the parse rules.\n\nIt would be good if auto-generation errored on useless allow ranges, but that will need wait since it does not impact production." }, { "commit": "144469b9772bad14466fcafc65edef58c5366755", "date": "2021-10-15 15:50:55 -0400", "subject": "Add const buffer functions to Pack type.", "body": "These allow packs to be created without allocating a buffer in the case that the buffer already exists or the data is in a global constant.\n\nAlso fix a rendering issue in hrnPackReadToStr()." }, { "commit": "66bfd1327e56f0f2de99fc6009431f3ee06ad6b8", "date": "2021-10-13 19:48:41 -0400", "subject": "Rename SSH connection control parameters in integration tests." }, { "commit": "447b24309d02938d04e036ec7814e75982210eb4", "date": "2021-10-13 19:43:40 -0400", "subject": "Update RHEL package URL." }, { "commit": "01b20724daf4c5cb25d6f636fb90456759773d22", "date": "2021-10-13 19:36:59 -0400", "subject": "Rename PostgreSQL pid file constants and tests." }, { "commit": "570162040864b8c56b236aa66d8f5c8d610b754b", "date": "2021-10-13 19:02:58 -0400", "subject": "Rename manifest file primary flag in tests." }, { "commit": "a44f9e373b47354a09bec0eaf2f3bf9e261c6941", "date": "2021-10-13 13:21:04 -0400", "subject": "Update Vagrantfile to Ubuntu 20.04." }, { "commit": "b16e827d69364408ea687a8e4b8894f7e889792e", "date": "2021-10-13 13:20:11 -0400", "subject": "Do not show output of PostgreSQL upgrade status check in user guide.", "body": "On some platforms the output may contain UTF-8 characters that the latex code is not prepared to handle.\n\nShowing the command is much more important than showing the output, so no big loss." }, { "commit": "5e84645ac030544b572036c08d055885d96d8905", "date": "2021-10-13 12:16:47 -0400", "subject": "Update comments referring to the PostgreSQL primary." }, { "commit": "430efff98a5b8dcf7c048f383abc12d9c0e5bbf0", "date": "2021-10-13 12:01:53 -0400", "subject": "Update documentation/links to main branch." }, { "commit": "1212668d5eff51756b0719b5296f7640e8096605", "date": "2021-10-13 11:43:14 -0400", "subject": "Update contributing.xml with rendering changes from def7d513.", "body": "Also update help.xml path missed in f4e1babf." }, { "commit": "90c73183ea5de6a63a23a9047ae2debb3f59b940", "date": "2021-10-13 09:37:03 -0400", "subject": "Add libc6-dbg required by updated valgrind to Vagrantfile/Dockerfile." }, { "commit": "c2d4552b7328489d703dc03defa769b1ccb8f739", "date": "2021-10-13 08:51:58 -0400", "subject": "Add debug options to code generation make in test.pl." }, { "commit": "bd91ebca759d2d6cfc2f7aa660366f5f7f09994a", "date": "2021-10-12 16:16:05 -0400", "subject": "Remove command overrides for output options.", "body": "The overrides are not needed since both commands require the same default and allow list." }, { "commit": "e8e346bc8738815b07bd80a37ace862fdab3dc1d", "date": "2021-10-12 08:53:12 -0400", "subject": "Remove command overrides for restore-only options.", "body": "The overrides are not needed since these options are only valid for one command." }, { "commit": "576b04763477877d4f2a61ad692703e20471901b", "date": "2021-10-11 16:25:36 -0400", "subject": "Invert required in set option to simplify generated rules." }, { "commit": "980b777a4a0cc07200ccb06cc55b89181101b266", "date": "2021-10-09 12:39:54 -0400", "subject": "Fix indentation." }, { "commit": "cc7f2eea900d0a7a429ffdc2cd45ead88c0298a9", "date": "2021-10-09 12:37:25 -0400", "subject": "Add assert in pckReadNext() to prevent reading into a field." }, { "commit": "610bfd736ef091a8298e5602d41aba86f10189bb", "date": "2021-10-09 12:34:45 -0400", "subject": "Increase tolerance for 0ms sleep in common/time test." }, { "commit": "7ab8dcbe6e007831e8a7f9f26e08f083c1026388", "date": "2021-10-09 12:15:19 -0400", "subject": "Read tag size in pckReadTagNext().", "body": "Rather than reading the size everywhere it is needed, get it when the tag is read, if it exists.\n\nThis simplifies logic around consuming the data when not needed. There are more use cases for this coming up." }, { "commit": "ed68792e765411a994d8ac79e4d047bbafc25582", "date": "2021-10-07 19:57:28 -0400", "subject": "Rename strNewN() to strNewZN().", "body": "Make the function name consistent with other functions that accept zero-terminated strings, e.g. strNewZ() and strCatZN()." }, { "commit": "b7e17d80ea02d70e56327828310c578af51795b5", "date": "2021-10-07 19:43:28 -0400", "subject": "More efficient memory allocation for Strings and String Variants.", "body": "The vast majority of Strings are never modified so for most cases allocate memory for the string with the object. This results in one allocation in most cases instead of two. Use strNew() if strCat*() functions are needed.\n\nUpdate varNewStr() in the same way since String Variants can never be modified. This results in one allocation in all cases instead of three. Also update varNewStrZ() to use STR() instead of strNewZ() to save two more allocations." }, { "commit": "208641ac7fd22f676a55e7305b3e69df574f36f8", "date": "2021-10-07 18:50:56 -0400", "subject": "Use constant string for user/group in performance/type test.", "body": "It is not safe to return strings created with STRDEF() from a function." }, { "commit": "74d3131830646c9f71ffa9847729cae40e3aa866", "date": "2021-10-07 14:58:11 -0400", "subject": "More efficient generation of diff/incr backup label." }, { "commit": "498902e885c9d7a44648ccc01b20454094d5b742", "date": "2021-10-07 12:18:24 -0400", "subject": "Allow \"global\" as a stanza prefix.", "body": "A stanza name like global_stanza was not allowed because the code was not selective enough about how a global section should be formatted.\r\n\r\nUpdate the config parser to correctly recognize global sections." }, { "commit": "338102861fd0ea4d2773b010dee34a39a96ad702", "date": "2021-10-07 11:01:48 -0400", "subject": "Improve instructions for rebuilding pgbackrest during stress testing." }, { "commit": "fb3f6928c9aef499938e195fbb612c1940a2dc19", "date": "2021-10-06 19:27:04 -0400", "subject": "Add configurable storage helpers to create repository storage.", "body": "Remove the hardcoded storage helpers from storageRepoGet() except for the the built-in Posix helper and the special remote helper.\n\nThe goal is to make storage driver development a bit easier by isolating as much of the code as possible into the driver module. This also makes coverage reporting much simpler for additional drivers since they do not need to provide coverage for storage/helper.\n\nConsolidate the CIFS tests into the Posix tests since CIFS is just a special case of the Posix.\n\nTest all storage features in the Posix test so that other storage driver tests do not need to provide coverage for storage/storage.\n\nRemove some dead code in the storage/s3 test." }, { "commit": "cfd823355af2ac99f30c5e1393a121c6dbf622b7", "date": "2021-10-06 12:38:56 -0400", "subject": "Refactor S3 storage driver for additional auth methods.", "body": "Currently only two auth methods are supported and a lot of refactoring is required to add a third one.\n\nDo the refactoring now to reduce noise in the commit that adds the third auth method." }, { "commit": "68c5f3eaf18fc9bd10fde15a323890bcdcbf4534", "date": "2021-10-05 17:59:05 -0400", "subject": "Allow link-map option to create new links.", "body": "Currently link-map only allows links that exist in the backup manifest to be remapped to a new destination.\r\n\r\nAllow link-map to create a new link as long as a valid path/file from the backup is referenced." }, { "commit": "f2aeb30fc706c04d6200cefeeb2645229a31ff69", "date": "2021-10-05 14:06:59 -0400", "subject": "Add state to ProtocolClient.", "body": "This is currently only useful for debugging, but in the future the state may be used for resetting the protocol when something goes wrong." }, { "commit": "2c65fed80f47124283d4f8be92f987dc55237f48", "date": "2021-10-05 12:29:16 -0400", "subject": "Add missing asserts and move temp mem context block." }, { "commit": "6af827cbb1e78cd4c5d649ed4cb24c49a7204b8f", "date": "2021-10-05 09:00:16 -0400", "subject": "Report original error and retries on local job failure.", "body": "The local process will retry jobs (e.g. backup file) but after a certain number of failures gives up. Previously, the last error was reported but generally the first error is far more valuable. The last error is likely to be a cascade failure such as the protocol being out of sync.\r\n\r\nReport the first error (and stack trace) and append the retry errors to the first error without stack trace information." }, { "commit": "34f78734325743b4e34cb39224852e6debf49750", "date": "2021-10-04 13:45:53 -0400", "subject": "Report backup file validation errors in backup.info.", "body": "Currently errors found during the backup are only available in text output when specifying --set.\r\n\r\nAdd a flag to backup.info that is available in both the text and json output when --set is not specified. This at least provides the basic info that an error was found in the cluster during the backup, though details are still only available as described above." }, { "commit": "57c62315465972f6b85558020198134e34cf2ee0", "date": "2021-10-02 17:27:33 -0400", "subject": "Add arm64 testing on Cirrus CI.", "body": "These tests run in a container without permissions to mount tempfs, so add an option to ci.pl to not create tempfs. Also add some packages not in the base image." }, { "commit": "f1ed8f0e5112d1a74d86168e67632be55eddb416", "date": "2021-10-02 16:29:31 -0400", "subject": "Sort WAL segment names when reporting duplicates.", "body": "Make the output consistent even when files are listed in a different order. This is purely for testing purposes, but there is no harm in consistent output.\n\nFound on arm64." }, { "commit": "71047a9d6d1eea71b3fbd430983541a54049cc69", "date": "2021-10-02 16:17:33 -0400", "subject": "Use strncpy() to limit characters copied to optionName.", "body": "Valgrind complained about uninitialized values on arm64 when comparing the reset prefix, probably because \"reset\" ended up being larger than the option name: Conditional jump or move depends on uninitialised value(s) at cfgParseOption (parse.c:568).\n\nCoverity complained because it could not verify the size of the string to be copied into optionName, probably because it does not understand the purpose of strSize(): You might overrun the 65-character fixed-size string \"optionName\" by copying the return value of \"strZ\" without checking the length.\n\nUse strncpy() even though we have already checked the size and make sure the string is terminated. Keep the size check because searching for truncated option names is not a good idea.\n\nThis is not a production bug since the code has not been released yet." }, { "commit": "b792a14cd7dbdfb61362700ffc5fc01997db890c", "date": "2021-10-01 18:23:03 -0400", "subject": "Use temp mem context when calling command handlers.", "body": "It is safer and more efficient to free memory after each handler completes.\n\nThe db command handlers use the server context so update them to use the top context." }, { "commit": "ae40ed6ec9cf77e577518b128518a5763767f589", "date": "2021-10-01 17:15:36 -0400", "subject": "Add jobRetry parameter to HRN_CFG_LOAD().", "body": "Allow the default of 0 to be overridden to test retry behavior for commands." }, { "commit": "136d309dd4bc1ada9f3d775f036b62292fda390b", "date": "2021-10-01 15:29:31 -0400", "subject": "Allow stack trace to be specified for errorInternalThrow().", "body": "This allows the stack trace to be set when an error is received by the protocol, rather than appending it to the message. Now these errors will look no different than any other error and the stack trace will be reported in the same way.\n\nOne immediate benefit is that test.pl --vm-out --log-level-test=debug will work for tests that check expect log results. Previously, the test would error at the first check because the stack trace included in the message would not match the expected log output." }, { "commit": "62f6fbe2a9ecd9bc48611ecfaf27a05b9f36a87d", "date": "2021-10-01 10:15:34 -0400", "subject": "Update file mode in info/manifest test to 0600.", "body": "0400 is not a very realistic mode. It may have become the default due to copy-pasting." }, { "commit": "0690cb25a077735780b2fe24343c946f8a4efbc6", "date": "2021-09-30 17:55:38 -0400", "subject": "Remove repository format 6 notes.", "body": "The notes have been moved to a Github project." }, { "commit": "376362475e3b3b13b70313d9002d4ff8b25b4b40", "date": "2021-09-30 16:15:45 -0400", "subject": "Move archive-header-check option to the archive reference section." }, { "commit": "cf1a57518fe3230886509f59fcec9c9a81e6513c", "date": "2021-09-30 14:29:49 -0400", "subject": "Refactor restoreManifestMap() to be driven by link-map.", "body": "This will allow new links to be added in a future commit. The current implementation is driven by the links that already exist in the manifest, which would make the new use case more complex to implement.\n\nAlso, add a more helpful error when a tablespace link is specified." }, { "commit": "d89a67776cfbb7b3047dbe297cc0c768e5c670e8", "date": "2021-09-30 13:39:29 -0400", "subject": "Refactor restoreManifestMap() tests in the command/restore unit.", "body": "Add test titles, new tests, and rearrange.\n\nAlso manifestTargetFindDefault(), which will soon be used by core code in a refactoring commit." }, { "commit": "7a53ba7c7f38bff2c9ef99c6ab58d22c59dd2290", "date": "2021-09-30 13:28:14 -0400", "subject": "Add note to comment for int64 typedef." }, { "commit": "815377cc6009c800b6a5fdc1fe98ddfceaaae824", "date": "2021-09-30 13:27:14 -0400", "subject": "Finalize catalog number for PostgreSQL 14 release." }, { "commit": "baf186bfb05ca683714a59485900892f5e7e8a1b", "date": "2021-09-29 12:03:01 -0400", "subject": "Fix comment typos." }, { "commit": "9e79f0e64b661e944ce2b3897c366feea1544ac2", "date": "2021-09-29 10:31:51 -0400", "subject": "Add recovery start time to online backup restore log.", "body": "This helps give an idea of how much recovery needs to be done to reach the end of the WAL stream and is easier to read than the backup label." }, { "commit": "9346895f5b61627b50d431f09447e08d8a50caa8", "date": "2021-09-29 09:58:47 -0400", "subject": "Rename page checksum error to error list in info text output.", "body": "\"error list\" makes it clearer that other errors may be reported. For example, if checksum-page is true in the manifest but no checksum-page-error list is provided then the error is in alignment, i.e. the file size is not a multiple of the page size, with allowances made for a valid-looking partial page at the end of the file.\r\n\r\nIt is still not possible to differentiate between alignment and page checksum errors in the output but this will be addressed in a future commit." }, { "commit": "b7ef12a76f219881d0b24c715592af96fe5c9b8f", "date": "2021-09-28 15:55:13 -0400", "subject": "Add hints to standby replay timeout message." }, { "commit": "096829b3b257444162417662612e80d8cb2ac6ec", "date": "2021-09-27 09:01:53 -0400", "subject": "Add repo-azure-uri-style option.", "body": "Azurite introduced a breaking change in 8f63964e to use automatically host-style URIs when the endpoint appears to be a multipart hostname.\n\nThis option allows the user to configure which style URI will be used, but changing the endpoint might cause breakage if Azurite decides to use a different style. Future changes to Azurite may also cause breakage." }, { "commit": "c8ea17c68f8fa72f2bf3b979539be3f709448493", "date": "2021-09-24 17:40:31 -0400", "subject": "Convert page checksum filter result to a pack.", "body": "The pack is both more compact and more efficient than a variant.\n\nAlso aggregate the page error info in the main process rather than in the filter to allow additional LSN filtering, to be added in a future commit." }, { "commit": "ac1f6db4a25520b1ee957b66925dde2e1ef156ab", "date": "2021-09-23 14:06:00 -0400", "subject": "Centralize and optimize tag stack management.", "body": "The push and pop code was duplicated in four places, so centralize the code into pckTagStackPop() and pckTagStackPush().\n\nAlso create a default bottom item for the stack to avoid allocating a list if there will only ever be the default container, which is very common. This avoids the extra time and memory to allocate a list." }, { "commit": "15e7ff10d3d6fe3570335a5abec5ff683c07e2e6", "date": "2021-09-23 08:31:32 -0400", "subject": "Add Pack pseudo-type.", "body": "Rather than working directly with Buffer types, define a new Pack pseudo-type that represents a Buffer containing a pack. This makes it clearer that a pack is being stored and allows stronger typing." }, { "commit": "131ac0ab5e98500569d4ff6985d31c55a9ef53b9", "date": "2021-09-22 11:18:12 -0400", "subject": "Rename pckReadNew()/pckWriteNew() to pckReadNewIo()/pckWriteNewIo().", "body": "These names more accurately describe the purpose of the constructors." }, { "commit": "0e76ccb5b7b2089f8d0300ab0086c454aebbbbbf", "date": "2021-09-22 10:48:21 -0400", "subject": "Convert filter param/result to Pack type.", "body": "The Pack type is more compact and flexible than the Variant type. The Pack type also allows binary data to be stored, which is useful for transferring the passphrase in the CipherBlock filter.\n\nThe primary purpose is to allow more (and more complex) result data to be returned efficiently from the PageChecksum filter. For now the PageChecksum filter still returns the original Variant. Converting the result data will be the subject of a future commit.\n\nAlso convert filter types to StringId." }, { "commit": "802373cb9df28384529fe5a7bd102bfe5c8f3911", "date": "2021-09-21 10:16:16 -0400", "subject": "Limit valgrind error output to the first error.", "body": "Generally the first error is the only important error. The rest simply lead to a lot of scrolling." }, { "commit": "473afce57bc7646c53bae4a6300b41b11b5b0357", "date": "2021-09-20 11:03:50 -0400", "subject": "Copy data page before verifying checksum.", "body": "Using UNCONSTIFY() is potentially dangerous since the buffer is modified while calculating the checksum, even though the page is reverted to the original state. Instead make a copy to ensure that the original data is never modified.\n\nThis requires the logic to be shuffled a bit since the copy cannot be made until we are sure the page is complete." }, { "commit": "0efb8adb9452cb8bc67ebaa8b6dca0d4a69c1682", "date": "2021-09-19 20:38:51 -0400", "subject": "Automatically include all PostgreSQL version interface files." }, { "commit": "95d814cf81b84d162f40717346c3ad0cb642f724", "date": "2021-09-19 20:32:27 -0400", "subject": "Specify size for helpData array." }, { "commit": "912a498b0bcc988bd5ebaaa22198956d752251bd", "date": "2021-09-11 16:07:59 -0400", "subject": "Skip comments when rendering help output.", "body": "Comments should not appear in the help. They are simply notes on implementation." }, { "commit": "c38d6926d6c9aa01b895a28e66fc0aa6965350a3", "date": "2021-09-09 08:48:45 -0400", "subject": "Revert Azurite version for testing to 3.14.0.", "body": "3.14.2 is causing breakage in the documentation. There is no obvious cause so for now just revert to the last working version." }, { "commit": "f4e1babf6b4ce7087ace8221cac7cadb51488f0e", "date": "2021-09-08 18:16:06 -0400", "subject": "Migrate command-line help generation to C.", "body": "Command-line help is now generated at build time so it does not need to be committed. This reduces churn on commits that add configuration and/or update the help.\n\nSince churn is no longer an issue, help.auto.c is bzip2 compressed to save space in the binary.\n\nThe Perl config parser (Data.pm) has been moved to doc/lib since the Perl build path is no longer required.\n\nLikewise doc/xml/reference.xml has been moved to src/build/help/help.xml since it is required at build time." }, { "commit": "def7d513cdd2d4579acf6e8c675a3d6f7da4f655", "date": "2021-09-08 17:35:45 -0400", "subject": "Eliminate linefeed formatting from documentation.", "body": "Linefeeds were originally used in the place of

tags to denote a paragraph. While much of the linefeed usage has been replaced over time, there were many places where it was still being used, especially in reference.xml. This made it difficult to get consistent formatting across different output types. In particular there were formatting issues in the command-line help because it is harder to audit than HTML or PDF.\n\nReplace linefeed formatting with proper

tags to make formatting more consistent.\n\nRemove double spaces in all text where

tags were added since it does not add churn.\n\nUpdate all