db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-02-03 13:21:32 +02:00
Code Issues Releases Activity

Remove internal repo-create command.

Browse Source 
This command was used by the Perl integration tests to create buckets for storage types that required it. Now that the integration tests are written in C they can simply use the same code to create buckets.

The command was also used in the documentation but there it seems more appropriate to use the corresponding vendor CLI.
This commit is contained in:
David Steele 2024-07-22 14:17:47 +07:00
parent 55ca41e137
commit ee70c2e26e
14 changed files with 56 additions and 320 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
doc/xml/user-guide.xml
View File

@ -337,6 +337,9 @@
/etc/postgresql-common/createcluster.conf && \
apt-get install -y --no-install-recommends postgresql-{[pg-version]} postgresql-{[pg-version-upgrade]} 2>&1
# Install Azure CLI
RUN apt-get install -y azure-cli
# Create an ssh key for root so all hosts can ssh to each other as root
RUN \ {[ssh-key-install]}
@ -402,6 +405,11 @@
# Install PostgreSQL
RUN yum install -y postgresql{[pg-version-nodot]}-server postgresql{[pg-version-upgrade-nodot]}-server
# Install Azure CLI
RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc
RUN dnf install -y https://packages.microsoft.com/config/rhel/8/packages-microsoft-prod.rpm
RUN dnf install -y azure-cli
# Create an ssh key for root so all hosts can ssh to each other as root
RUN \ {[ssh-key-install]}
@ -744,7 +752,11 @@
</execute>
<execute user="{[azure-setup-user]}" if="'{[azure-setup-create-container]}' eq 'y'" show='n'>
<exe-cmd>{[project-exe]} --repo={[azure-setup-repo-id]} repo-create</exe-cmd>
<exe-cmd>
bash -c 'export AZURE_CLI_DISABLE_CONNECTION_VERIFICATION=1;az storage container create -n {[azure-container]}
--connection-string "DefaultEndpointsProtocol=https;AccountName={[azure-account]};AccountKey={[azure-key]}"
2>&amp;1'
</exe-cmd>
</execute>
</execute-list>
@ -790,15 +802,22 @@
</backrest-config>
<execute-list if="'{[s3-local]}' eq 'y'" host="{[s3-setup-host]}" show="n">
<title>Create the bucket</title>
<title>Create DNS entry</title>
<!-- Set host entries to redirect AWS to local s3 server -->
<execute user="root" user-force="y" show="n">
<exe-cmd>echo "{[host-s3-ip]} {[s3-bucket]}.{[s3-endpoint]} {[s3-endpoint]}" | tee -a /etc/hosts</exe-cmd>
</execute>
</execute-list>
<execute user="{[s3-setup-user]}" if="'{[s3-setup-create-bucket]}' eq 'y'" show='n'>
<exe-cmd>{[project-exe]} --repo={[s3-setup-repo-id]} repo-create</exe-cmd>
<execute-list if="'{[s3-local]}' eq 'y' &amp;&amp; '{[s3-setup-create-bucket]}' eq 'y'" host="{[host-s3]}" show="n">
<title>Create bucket</title>
<execute user="root" user-force="y">
<exe-cmd>mc --insecure alias set s3 https://127.0.0.1 {[s3-key]} {[s3-key-secret]}</exe-cmd>
</execute>
<execute user="root" user-force="y">
<exe-cmd>mc --insecure mb s3/{[s3-bucket]}</exe-cmd>
</execute>
</execute-list>

1
src/Makefile.in
View File

@ -81,7 +81,6 @@ SRCS = \
command/lock.c \
command/manifest/manifest.c \
command/repo/common.c \
command/repo/create.c \
command/repo/get.c \
command/repo/ls.c \
command/repo/put.c \

26
src/build/config/config.yaml
View File

@ -93,12 +93,6 @@ command:
log-file: false
log-level-default: DEBUG
repo-create:
command-role:
remote: {}
internal: true
log-file: false
repo-get:
command-role:
remote: {}
@ -386,8 +380,6 @@ option:
info:
required: false
manifest: {}
repo-create:
required: false
repo-get:
required: false
repo-ls:
@ -529,7 +521,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -572,7 +563,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -692,7 +682,6 @@ option:
expire: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -716,7 +705,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -807,7 +795,6 @@ option:
archive-push: {}
backup: {}
check: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -893,7 +880,6 @@ option:
backup: {}
check: {}
expire: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -934,7 +920,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -1093,7 +1078,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -1681,10 +1665,6 @@ option:
command-role:
main: {}
remote: {}
repo-create:
command-role:
main: {}
remote: {}
repo-get:
command-role:
main: {}
@ -1743,7 +1723,6 @@ option:
internal: true
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}
@ -1806,10 +1785,6 @@ option:
command-role:
main: {}
remote: {}
repo-create:
command-role:
main: {}
remote: {}
repo-get:
command-role:
main: {}
@ -2137,7 +2112,6 @@ option:
check: {}
info: {}
manifest: {}
repo-create: {}
repo-get: {}
repo-ls: {}
repo-put: {}

8
src/build/help/help.xml
View File

@ -2260,14 +2260,6 @@
</option-list>
</command>
<command id="repo-create" name="Create Repository">
<summary>Create the repository.</summary>
<text>
<p>Create the repository (path, bucket, etc.) on the default repository. If more than one repository is configured, the command will default to the highest priority repository (e.g. <id>repo1</id>) unless the <br-option>{[dash]}-repo</br-option> option is specified. If <setting>repo-type=s3</setting> the bucket will be created.</p>
</text>
</command>
<command id="repo-get" name="Repository Get">
<summary>Get a file from a repository.</summary>

62
src/command/repo/create.c
View File

@ -1,62 +0,0 @@
/***********************************************************************************************************************************
Repository Create Command
***********************************************************************************************************************************/
#include "build.auto.h"
#include "common/debug.h"
#include "common/log.h"
#include "common/memContext.h"
#include "common/type/json.h"
#include "config/config.h"
#include "storage/azure/storage.intern.h"
#include "storage/gcs/storage.intern.h"
#include "storage/helper.h"
#include "storage/s3/storage.intern.h"
/**********************************************************************************************************************************/
FN_EXTERN void
cmdRepoCreate(void)
{
FUNCTION_LOG_VOID(logLevelDebug);
MEM_CONTEXT_TEMP_BEGIN()
{
switch (storageType(storageRepo()))
{
case STORAGE_AZURE_TYPE:
{
storageAzureRequestP(
(StorageAzure *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR,
.query = httpQueryAdd(httpQueryNewP(), AZURE_QUERY_RESTYPE_STR, AZURE_QUERY_VALUE_CONTAINER_STR));
break;
}
case STORAGE_GCS_TYPE:
{
storageGcsRequestP(
(StorageGcs *)storageDriver(storageRepoWrite()), HTTP_VERB_POST_STR, .noBucket = true,
.content = BUFSTR(
jsonWriteResult(
jsonWriteObjectEnd(
jsonWriteStr(
jsonWriteKeyZ(
jsonWriteObjectBegin(
jsonWriteNewP()), GCS_JSON_NAME), cfgOptionStr(cfgOptRepoGcsBucket))))));
break;
}
case STORAGE_S3_TYPE:
storageS3RequestP((StorageS3 *)storageDriver(storageRepoWrite()), HTTP_VERB_PUT_STR, FSLASH_STR);
break;
// Other storage types do not require the repo to be created
default:
break;
}
}
MEM_CONTEXT_TEMP_END();
FUNCTION_LOG_RETURN_VOID();
}

13
src/command/repo/create.h
View File

@ -1,13 +0,0 @@
/***********************************************************************************************************************************
Repository Create Command
***********************************************************************************************************************************/
#ifndef COMMAND_REPO_CREATE_H
#define COMMAND_REPO_CREATE_H
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
// Create the repository (create path, bucket, etc.)
FN_EXTERN void cmdRepoCreate(void);
#endif

4
src/config/config.auto.h
View File

@ -18,7 +18,6 @@ Command constants
#define CFGCMD_HELP "help"
#define CFGCMD_INFO "info"
#define CFGCMD_MANIFEST "manifest"
#define CFGCMD_REPO_CREATE "repo-create"
#define CFGCMD_REPO_GET "repo-get"
#define CFGCMD_REPO_LS "repo-ls"
#define CFGCMD_REPO_PUT "repo-put"
@ -34,7 +33,7 @@ Command constants
#define CFGCMD_VERIFY "verify"
#define CFGCMD_VERSION "version"
#define CFG_COMMAND_TOTAL 24
#define CFG_COMMAND_TOTAL 23
/***********************************************************************************************************************************
Option group constants
@ -353,7 +352,6 @@ typedef enum
cfgCmdHelp,
cfgCmdInfo,
cfgCmdManifest,
cfgCmdRepoCreate,
cfgCmdRepoGet,
cfgCmdRepoLs,
cfgCmdRepoPut,

161
src/config/parse.auto.c.inc
View File

File diff suppressed because it is too large Load Diff

7
src/main.c
View File

@ -23,7 +23,6 @@ Main
#include "command/lock.h"
#include "command/manifest/manifest.h"
#include "command/remote/remote.h"
#include "command/repo/create.h"
#include "command/repo/get.h"
#include "command/repo/ls.h"
#include "command/repo/put.h"
@ -220,12 +219,6 @@ main(int argListSize, const char *argList[])
cmdManifest();
break;
// Repository create command
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdRepoCreate:
cmdRepoCreate();
break;
// Repository get file command
// -----------------------------------------------------------------------------------------------------------------
case cfgCmdRepoGet:

1
src/meson.build
View File

@ -149,7 +149,6 @@ src_pgbackrest = [
'command/lock.c',
'command/manifest/manifest.c',
'command/repo/common.c',
'command/repo/create.c',
'command/repo/get.c',
'command/repo/ls.c',
'command/repo/put.c',

8
test/code-count/file-type.yaml
View File

@ -1099,14 +1099,6 @@ src/command/repo/common.h:
class: core
type: c/h
src/command/repo/create.c:
class: core
type: c
src/command/repo/create.h:
class: core
type: c/h
src/command/repo/get.c:
class: core
type: c

7
test/define.yaml
View File

@ -960,7 +960,7 @@ unit:
# ----------------------------------------------------------------------------------------------------------------------------
- name: repo
total: 4
total: 3
coverage:
- command/repo/common
@ -969,11 +969,6 @@ unit:
- command/repo/put
- command/repo/rm
include:
# command/repo/create is currently for testing purposes only so coverage is not provided except in integration. In the
# future this will probably be rolled into a custom object store server implementation.
- command/repo/create
# ----------------------------------------------------------------------------------------------------------------------------
- name: verify
total: 12

38
test/src/common/harnessHost.c
View File

@ -8,14 +8,15 @@ Host Harness
#include "common/crypto/common.h"
#include "common/error/retry.h"
#include "common/io/io.h"
#include "common/type/json.h"
#include "common/wait.h"
#include "config/config.h"
#include "postgres/interface.h"
#include "postgres/version.h"
#include "storage/azure/storage.h"
#include "storage/gcs/storage.h"
#include "storage/azure/storage.intern.h"
#include "storage/gcs/storage.intern.h"
#include "storage/posix/storage.h"
#include "storage/s3/storage.h"
#include "storage/s3/storage.intern.h"
#include "storage/sftp/storage.h"
#include "common/harnessDebug.h"
@ -1289,11 +1290,34 @@ hrnHostBuild(const int line, const HrnHostTestDefine *const testMatrix, const si
// Write pgBackRest configuration for hosts
hrnHostConfigUpdateP();
// Create the repo for object stores
if (hrnHostLocal.storage == STORAGE_AZURE_TYPE || hrnHostLocal.storage == STORAGE_GCS_TYPE ||
hrnHostLocal.storage == STORAGE_S3_TYPE)
// Create the bucket/container for object stores
switch (hrnHostLocal.storage)
{
hrnHostExecBrP(repo, CFGCMD_REPO_CREATE, .option = "--repo=1");
case STORAGE_AZURE_TYPE:
{
storageAzureRequestP(
(StorageAzure *)storageDriver(hrnHostRepo1Storage(repo)), HTTP_VERB_PUT_STR,
.query = httpQueryAdd(httpQueryNewP(), AZURE_QUERY_RESTYPE_STR, AZURE_QUERY_VALUE_CONTAINER_STR));
break;
}
case STORAGE_GCS_TYPE:
{
JsonWrite *const json = jsonWriteObjectBegin(jsonWriteNewP());
jsonWriteStr(jsonWriteKeyZ(json, GCS_JSON_NAME), STRDEF(HRN_HOST_GCS_BUCKET));
jsonWriteObjectEnd(json);
storageGcsRequestP(
(StorageGcs *)storageDriver(hrnHostRepo1Storage(repo)), HTTP_VERB_POST_STR, .noBucket = true,
.content = BUFSTR(jsonWriteResult(json)));
break;
}
case STORAGE_S3_TYPE:
storageS3RequestP((StorageS3 *)storageDriver(hrnHostRepo1Storage(repo)), HTTP_VERB_PUT_STR, FSLASH_STR);
break;
}
FUNCTION_HARNESS_RETURN_VOID();

13
test/src/module/command/repoTest.c
View File

@ -19,19 +19,6 @@ testRun(void)
{
FUNCTION_HARNESS_VOID();
// *****************************************************************************************************************************
if (testBegin("cmdRepoCreate()"))
{
StringList *argList = strLstNew();
hrnCfgArgRawZ(argList, cfgOptRepoPath, TEST_PATH "/repo");
HRN_CFG_LOAD(cfgCmdRepoCreate, argList);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("noop on non-S3 storage");
TEST_RESULT_VOID(cmdRepoCreate(), "repo create");
}
// *****************************************************************************************************************************
if (testBegin("cmdStorageList() and storageListRender()"))
{