db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-01-18 04:58:51 +02:00
Code Issues Releases Activity

Improve interface handling in storage module.

Browse Source 
Make the interface object the parent of the driver object rather than the interface being allocated directly in the driver object.

The prior method was more efficient when mem contexts had a much higher cost. Now mem contexts are cheap so it makes more sense to structure the objects in a way that works better with mem context auditing. This also means the mem context does not need to be stored separately since it can be extracted directly from the interface object.

There are other areas that need to get the same improvement before the specialized objMoveContext() and objFreeContext() functions can be removed.
This commit is contained in:
David Steele 2023-03-07 11:27:41 +07:00
parent 120a49b659
commit f6e307365f
20 changed files with 239 additions and 238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/storage/azure/read.c
View File

@ -123,13 +123,13 @@ storageReadAzureNew(
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageRead *this = NULL;
StorageReadAzure *this = NULL;
OBJ_NEW_BEGIN(StorageReadAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StorageReadAzure, .childQty = MEM_CONTEXT_QTY_MAX)
{
StorageReadAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadAzure);
this = OBJ_NEW_ALLOC();
*driver = (StorageReadAzure)
*this = (StorageReadAzure)
{
.storage = storage,
@ -149,10 +149,8 @@ storageReadAzureNew(
},
},
};
this = storageReadNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_READ, this);
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
}

32
src/storage/azure/storage.c
View File

@ -734,13 +734,13 @@ storageAzureNew(
ASSERT(key != NULL);
ASSERT(blockSize != 0);
Storage *this = NULL;
StorageAzure *this = NULL;
OBJ_NEW_BEGIN(StorageAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StorageAzure, .childQty = MEM_CONTEXT_QTY_MAX)
{
StorageAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageAzure);
this = OBJ_NEW_ALLOC();
*driver = (StorageAzure)
*this = (StorageAzure)
{
.interface = storageInterfaceAzure,
.container = strDup(container),
@ -754,32 +754,30 @@ storageAzureNew(
// Store shared key or parse sas query
if (keyType == storageAzureKeyTypeShared)
driver->sharedKey = bufNewDecode(encodingBase64, key);
this->sharedKey = bufNewDecode(encodingBase64, key);
else
driver->sasKey = httpQueryNewStr(key);
this->sasKey = httpQueryNewStr(key);
// Create the http client used to service requests
driver->httpClient = httpClientNew(
this->httpClient = httpClientNew(
tlsClientNewP(
sckClientNew(driver->host, port, timeout, timeout), driver->host, timeout, timeout, verifyPeer, .caFile = caFile,
sckClientNew(this->host, port, timeout, timeout), this->host, timeout, timeout, verifyPeer, .caFile = caFile,
.caPath = caPath),
timeout);
// Create list of redacted headers
driver->headerRedactList = strLstNew();
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(driver->headerRedactList, HTTP_HEADER_DATE_STR);
this->headerRedactList = strLstNew();
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(this->headerRedactList, HTTP_HEADER_DATE_STR);
// Create list of redacted query keys
driver->queryRedactList = strLstNew();
strLstAdd(driver->queryRedactList, AZURE_QUERY_SIG_STR);
this->queryRedactList = strLstNew();
strLstAdd(this->queryRedactList, AZURE_QUERY_SIG_STR);
// Generate starting file id
cryptoRandomBytes((unsigned char *)&driver->fileId, sizeof(driver->fileId));
this = storageNew(STORAGE_AZURE_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
cryptoRandomBytes((unsigned char *)&this->fileId, sizeof(this->fileId));
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_AZURE_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
}

14
src/storage/azure/write.c
View File

@ -262,7 +262,7 @@ storageWriteAzureClose(THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN StorageWrite *
storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId, size_t blockSize)
storageWriteAzureNew(StorageAzure *const storage, const String *const name, const uint64_t fileId, const size_t blockSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_AZURE, storage);
@ -274,13 +274,13 @@ storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId,
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageWrite *this = NULL;
StorageWriteAzure *this = NULL;
OBJ_NEW_BEGIN(StorageWriteAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StorageWriteAzure, .childQty = MEM_CONTEXT_QTY_MAX)
{
StorageWriteAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteAzure);
this = OBJ_NEW_ALLOC();
*driver = (StorageWriteAzure)
*this = (StorageWriteAzure)
{
.storage = storage,
.fileId = fileId,
@ -304,10 +304,8 @@ storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId,
},
},
};
this = storageWriteNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
}

10
src/storage/gcs/read.c
View File

@ -130,13 +130,13 @@ storageReadGcsNew(
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageRead *this = NULL;
StorageReadGcs *this = NULL;
OBJ_NEW_BEGIN(StorageReadGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
{
StorageReadGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadGcs);
this = OBJ_NEW_ALLOC();
*driver = (StorageReadGcs)
*this = (StorageReadGcs)
{
.storage = storage,
@ -156,10 +156,8 @@ storageReadGcsNew(
},
},
};
this = storageReadNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_READ, this);
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
}

54
src/storage/gcs/storage.c
View File

@ -946,9 +946,9 @@ static const StorageInterface storageInterfaceGcs =
FN_EXTERN Storage *
storageGcsNew(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
StorageGcsKeyType keyType, const String *key, size_t chunkSize, const String *endpoint, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath)
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket,
const StorageGcsKeyType keyType, const String *const key, const size_t chunkSize, const String *const endpoint,
const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -970,13 +970,13 @@ storageGcsNew(
ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL);
ASSERT(chunkSize != 0);
Storage *this = NULL;
StorageGcs *this = NULL;
OBJ_NEW_BEGIN(StorageGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
{
StorageGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageGcs);
this = OBJ_NEW_ALLOC();
*driver = (StorageGcs)
*this = (StorageGcs)
{
.interface = storageInterfaceGcs,
.write = write,
@ -991,11 +991,11 @@ storageGcsNew(
// Auto authentication for GCE instances
case storageGcsKeyTypeAuto:
{
driver->authUrl = httpUrlNewParseP(
this->authUrl = httpUrlNewParseP(
STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
.type = httpProtocolTypeHttp);
driver->authClient = httpClientNew(
sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout, timeout), timeout);
this->authClient = httpClientNew(
sckClientNew(httpUrlHost(this->authUrl), httpUrlPort(this->authUrl), timeout, timeout), timeout);
break;
}
@ -1004,18 +1004,18 @@ storageGcsNew(
case storageGcsKeyTypeService:
{
KeyValue *kvKey = varKv(jsonToVar(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key)))));
driver->credential = varStr(kvGet(kvKey, GCS_JSON_CLIENT_EMAIL_VAR));
driver->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR));
this->credential = varStr(kvGet(kvKey, GCS_JSON_CLIENT_EMAIL_VAR));
this->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR));
const String *const uri = varStr(kvGet(kvKey, GCS_JSON_TOKEN_URI_VAR));
CHECK(FormatError, driver->credential != NULL && driver->privateKey != NULL && uri != NULL, "credentials missing");
CHECK(FormatError, this->credential != NULL && this->privateKey != NULL && uri != NULL, "credentials missing");
driver->authUrl = httpUrlNewParseP(uri, .type = httpProtocolTypeHttps);
this->authUrl = httpUrlNewParseP(uri, .type = httpProtocolTypeHttps);
driver->authClient = httpClientNew(
this->authClient = httpClientNew(
tlsClientNewP(
sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout, timeout),
httpUrlHost(driver->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
sckClientNew(httpUrlHost(this->authUrl), httpUrlPort(this->authUrl), timeout, timeout),
httpUrlHost(this->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
timeout);
break;
@ -1023,33 +1023,31 @@ storageGcsNew(
// Store the authentication token
case storageGcsKeyTypeToken:
driver->token = strDup(key);
this->token = strDup(key);
break;
}
// Parse the endpoint to extract the host and port
HttpUrl *url = httpUrlNewParseP(endpoint, .type = httpProtocolTypeHttps);
driver->endpoint = httpUrlHost(url);
this->endpoint = httpUrlHost(url);
// Create the http client used to service requests
driver->httpClient = httpClientNew(
this->httpClient = httpClientNew(
tlsClientNewP(
sckClientNew(driver->endpoint, httpUrlPort(url), timeout, timeout), driver->endpoint, timeout, timeout, verifyPeer,
sckClientNew(this->endpoint, httpUrlPort(url), timeout, timeout), this->endpoint, timeout, timeout, verifyPeer,
.caFile = caFile, .caPath = caPath),
timeout);
// Create list of redacted headers
driver->headerRedactList = strLstNew();
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(driver->headerRedactList, GCS_HEADER_UPLOAD_ID_STR);
this->headerRedactList = strLstNew();
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(this->headerRedactList, GCS_HEADER_UPLOAD_ID_STR);
// Create list of redacted query keys
driver->queryRedactList = strLstNew();
strLstAdd(driver->queryRedactList, GCS_QUERY_UPLOAD_ID_STR);
this = storageNew(STORAGE_GCS_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
this->queryRedactList = strLstNew();
strLstAdd(this->queryRedactList, GCS_QUERY_UPLOAD_ID_STR);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_GCS_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
}

14
src/storage/gcs/write.c
View File

@ -321,7 +321,7 @@ storageWriteGcsClose(THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN StorageWrite *
storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
storageWriteGcsNew(StorageGcs *const storage, const String *const name, const size_t chunkSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_GCS, storage);
@ -332,13 +332,13 @@ storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageWrite *this = NULL;
StorageWriteGcs *this = NULL;
OBJ_NEW_BEGIN(StorageWriteGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StorageWriteGcs, .childQty = MEM_CONTEXT_QTY_MAX)
{
StorageWriteGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteGcs);
this = OBJ_NEW_ALLOC();
*driver = (StorageWriteGcs)
*this = (StorageWriteGcs)
{
.storage = storage,
.chunkSize = chunkSize,
@ -361,10 +361,8 @@ storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
},
},
};
this = storageWriteNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
}

10
src/storage/posix/read.c
View File

@ -220,13 +220,13 @@ storageReadPosixNew(
ASSERT(name != NULL);
StorageRead *this = NULL;
StorageReadPosix *this = NULL;
OBJ_NEW_BEGIN(StorageReadPosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{
StorageReadPosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadPosix);
this = OBJ_NEW_ALLOC();
*driver = (StorageReadPosix)
*this = (StorageReadPosix)
{
.storage = storage,
.fd = -1,
@ -254,10 +254,8 @@ storageReadPosixNew(
},
},
};
this = storageReadNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_READ, this);
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
}

20
src/storage/posix/storage.c
View File

@ -590,8 +590,8 @@ static const StorageInterface storageInterfacePosix =
FN_EXTERN Storage *
storagePosixNewInternal(
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
StoragePathExpressionCallback pathExpressionFunction, bool pathSync)
const StringId type, const String *const path, const mode_t modeFile, const mode_t modePath, const bool write,
StoragePathExpressionCallback pathExpressionFunction, const bool pathSync)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING_ID, type);
@ -612,32 +612,30 @@ storagePosixNewInternal(
userInit();
// Create the object
Storage *this = NULL;
StoragePosix *this = NULL;
OBJ_NEW_BEGIN(StoragePosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StoragePosix, .childQty = MEM_CONTEXT_QTY_MAX)
{
StoragePosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StoragePosix);
this = OBJ_NEW_ALLOC();
*driver = (StoragePosix)
*this = (StoragePosix)
{
.interface = storageInterfacePosix,
};
// Disable path sync when not supported
if (!pathSync)
driver->interface.pathSync = NULL;
this->interface.pathSync = NULL;
// If this is a posix driver then add link features
if (type == STORAGE_POSIX_TYPE)
driver->interface.feature |=
this->interface.feature |=
1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync |
1 << storageFeatureInfoDetail;
this = storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
FUNCTION_LOG_RETURN(STORAGE, storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, this, this->interface));
}
FN_EXTERN Storage *

14
src/storage/posix/write.c
View File

@ -245,13 +245,13 @@ storageWritePosixNew(
ASSERT(modeFile != 0);
ASSERT(modePath != 0);
StorageWrite *this = NULL;
StorageWritePosix *this = NULL;
OBJ_NEW_BEGIN(StorageWritePosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
OBJ_NEW_BEGIN(StorageWritePosix, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
{
StorageWritePosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWritePosix);
this = OBJ_NEW_ALLOC();
*driver = (StorageWritePosix)
*this = (StorageWritePosix)
{
.storage = storage,
.path = strPath(name),
@ -283,11 +283,9 @@ storageWritePosixNew(
};
// Create temp file name
driver->nameTmp = atomic ? strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strZ(name)) : driver->interface.name;
this = storageWriteNew(driver, &driver->interface);
this->nameTmp = atomic ? strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strZ(name)) : this->interface.name;
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
}

27
src/storage/read.c
View File

@ -14,7 +14,7 @@ Object type
struct StorageRead
{
StorageReadPub pub; // Publicly accessible variables
void *driver;
void *driver; // Driver
};
/***********************************************************************************************************************************
@ -27,7 +27,7 @@ Macros for function logging
/**********************************************************************************************************************************/
FN_EXTERN StorageRead *
storageReadNew(void *driver, const StorageReadInterface *interface)
storageReadNew(void *const driver, const StorageReadInterface *const interface)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM_P(VOID, driver);
@ -41,18 +41,21 @@ storageReadNew(void *driver, const StorageReadInterface *interface)
StorageRead *this = NULL;
this = memNew(sizeof(StorageRead));
*this = (StorageRead)
OBJ_NEW_BEGIN(StorageRead, .childQty = MEM_CONTEXT_QTY_MAX)
{
.pub =
this = OBJ_NEW_ALLOC();
*this = (StorageRead)
{
.memContext = memContextCurrent(),
.interface = interface,
.io = ioReadNew(driver, interface->ioInterface),
},
.driver = driver,
};
.pub =
{
.interface = interface,
.io = ioReadNew(driver, interface->ioInterface),
},
.driver = objMove(driver, objMemContext(this)),
};
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_READ, this);
}

5
src/storage/read.h
View File

@ -20,7 +20,7 @@ Functions
FN_INLINE_ALWAYS StorageRead *
storageReadMove(StorageRead *const this, MemContext *const parentNew)
{
return objMoveContext(this, parentNew);
return objMove(this, parentNew);
}
/***********************************************************************************************************************************
@ -28,7 +28,6 @@ Getters/Setters
***********************************************************************************************************************************/
typedef struct StorageReadPub
{
MemContext *memContext; // Mem context
const StorageReadInterface *interface; // File data (name, driver type, etc.)
IoRead *io; // Read interface
} StorageReadPub;
@ -81,7 +80,7 @@ Destructor
FN_INLINE_ALWAYS void
storageReadFree(StorageRead *const this)
{
objFreeContext(this);
objFree(this);
}
/***********************************************************************************************************************************

22
src/storage/remote/storage.c
View File

@ -470,8 +470,8 @@ static const StorageInterface storageInterfaceRemote =
FN_EXTERN Storage *
storageRemoteNew(
mode_t modeFile, mode_t modePath, bool write, StoragePathExpressionCallback pathExpressionFunction, ProtocolClient *client,
unsigned int compressLevel)
const mode_t modeFile, const mode_t modePath, const bool write, StoragePathExpressionCallback pathExpressionFunction,
ProtocolClient *const client, const unsigned int compressLevel)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(MODE, modeFile);
@ -486,26 +486,25 @@ storageRemoteNew(
ASSERT(modePath != 0);
ASSERT(client != NULL);
Storage *this = NULL;
StorageRemote *this = NULL;
const String *path = NULL;
OBJ_NEW_BEGIN(StorageRemote, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
{
StorageRemote *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageRemote);
this = OBJ_NEW_ALLOC();
*driver = (StorageRemote)
*this = (StorageRemote)
{
.client = client,
.compressLevel = compressLevel,
.interface = storageInterfaceRemote,
};
const String *path = NULL;
// Get storage features from the remote
MEM_CONTEXT_TEMP_BEGIN()
{
// Execute command and get result
PackRead *result = protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
PackRead *result = protocolClientExecute(this->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
// Get path in parent context
MEM_CONTEXT_PRIOR_BEGIN()
@ -514,13 +513,12 @@ storageRemoteNew(
}
MEM_CONTEXT_PRIOR_END();
driver->interface.feature = pckReadU64P(result);
this->interface.feature = pckReadU64P(result);
}
MEM_CONTEXT_TEMP_END();
this = storageNew(STORAGE_REMOTE_TYPE, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
FUNCTION_LOG_RETURN(
STORAGE, storageNew(STORAGE_REMOTE_TYPE, path, modeFile, modePath, write, pathExpressionFunction, this, this->interface));
}

10
src/storage/s3/read.c
View File

@ -126,13 +126,13 @@ storageReadS3New(
ASSERT(name != NULL);
ASSERT(limit == NULL || varUInt64(limit) > 0);
StorageRead *this = NULL;
StorageReadS3 *this = NULL;
OBJ_NEW_BEGIN(StorageReadS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
{
StorageReadS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadS3);
this = OBJ_NEW_ALLOC();
*driver = (StorageReadS3)
*this = (StorageReadS3)
{
.storage = storage,
@ -152,10 +152,8 @@ storageReadS3New(
},
},
};
this = storageReadNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_READ, this);
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
}

67
src/storage/s3/storage.c
View File

@ -1093,11 +1093,12 @@ static const StorageInterface storageInterfaceS3 =
FN_EXTERN Storage *
storageS3New(
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
const String *secretAccessKey, const String *securityToken, const String *const kmsKeyId, const String *credRole,
const String *const webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
const String *caFile, const String *caPath)
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket,
const String *const endPoint, const StorageS3UriStyle uriStyle, const String *const region, const StorageS3KeyType keyType,
const String *const accessKey, const String *const secretAccessKey, const String *const securityToken,
const String *const kmsKeyId, const String *const credRole, const String *const webIdToken, const size_t partSize,
const String *host, const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile,
const String *const caPath)
{
FUNCTION_LOG_BEGIN(logLevelDebug);
FUNCTION_LOG_PARAM(STRING, path);
@ -1129,13 +1130,13 @@ storageS3New(
ASSERT(region != NULL);
ASSERT(partSize != 0);
Storage *this = NULL;
StorageS3 *this = NULL;
OBJ_NEW_BEGIN(StorageS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
OBJ_NEW_BEGIN(StorageS3, .childQty = MEM_CONTEXT_QTY_MAX)
{
StorageS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageS3);
this = OBJ_NEW_ALLOC();
*driver = (StorageS3)
*this = (StorageS3)
{
.interface = storageInterfaceS3,
.bucket = strDup(bucket),
@ -1154,26 +1155,26 @@ storageS3New(
// Create the HTTP client used to service requests
if (host == NULL)
host = driver->bucketEndpoint;
host = this->bucketEndpoint;
driver->httpClient = httpClientNew(
this->httpClient = httpClientNew(
tlsClientNewP(
sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
timeout);
// Initialize authentication
switch (driver->keyType)
switch (this->keyType)
{
// Create the HTTP client used to retrieve temporary security credentials
case storageS3KeyTypeAuto:
{
ASSERT(accessKey == NULL && secretAccessKey == NULL && securityToken == NULL);
driver->credRole = strDup(credRole);
driver->credHost = S3_CREDENTIAL_HOST_STR;
driver->credExpirationTime = time(NULL);
driver->credHttpClient = httpClientNew(
sckClientNew(driver->credHost, S3_CREDENTIAL_PORT, timeout, timeout), timeout);
this->credRole = strDup(credRole);
this->credHost = S3_CREDENTIAL_HOST_STR;
this->credExpirationTime = time(NULL);
this->credHttpClient = httpClientNew(
sckClientNew(this->credHost, S3_CREDENTIAL_PORT, timeout, timeout), timeout);
break;
}
@ -1185,13 +1186,13 @@ storageS3New(
ASSERT(credRole != NULL);
ASSERT(webIdToken != NULL);
driver->credRole = strDup(credRole);
driver->webIdToken = strDup(webIdToken);
driver->credHost = S3_STS_HOST_STR;
driver->credExpirationTime = time(NULL);
driver->credHttpClient = httpClientNew(
this->credRole = strDup(credRole);
this->webIdToken = strDup(webIdToken);
this->credHost = S3_STS_HOST_STR;
this->credExpirationTime = time(NULL);
this->credHttpClient = httpClientNew(
tlsClientNewP(
sckClientNew(driver->credHost, S3_STS_PORT, timeout, timeout), driver->credHost, timeout, timeout, true,
sckClientNew(this->credHost, S3_STS_PORT, timeout, timeout), this->credHost, timeout, timeout, true,
.caFile = caFile, .caPath = caPath),
timeout);
@ -1201,26 +1202,24 @@ storageS3New(
// Set shared key credentials
default:
{
ASSERT(driver->keyType == storageS3KeyTypeShared);
ASSERT(this->keyType == storageS3KeyTypeShared);
ASSERT(accessKey != NULL && secretAccessKey != NULL);
driver->accessKey = strDup(accessKey);
driver->secretAccessKey = strDup(secretAccessKey);
driver->securityToken = strDup(securityToken);
this->accessKey = strDup(accessKey);
this->secretAccessKey = strDup(secretAccessKey);
this->securityToken = strDup(securityToken);
break;
}
}
// Create list of redacted headers
driver->headerRedactList = strLstNew();
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(driver->headerRedactList, S3_HEADER_DATE_STR);
strLstAdd(driver->headerRedactList, S3_HEADER_TOKEN_STR);
this = storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
this->headerRedactList = strLstNew();
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
strLstAdd(this->headerRedactList, S3_HEADER_DATE_STR);
strLstAdd(this->headerRedactList, S3_HEADER_TOKEN_STR);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
}

12
src/storage/s3/write.c
View File

@ -268,7 +268,7 @@ storageWriteS3Close(THIS_VOID)
/**********************************************************************************************************************************/
FN_EXTERN StorageWrite *
storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
storageWriteS3New(StorageS3 *const storage, const String *const name, const size_t partSize)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STORAGE_S3, storage);
@ -278,13 +278,13 @@ storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
ASSERT(storage != NULL);
ASSERT(name != NULL);
StorageWrite *this = NULL;
StorageWriteS3 *this = NULL;
OBJ_NEW_BEGIN(StorageWriteS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
{
StorageWriteS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteS3);
this = OBJ_NEW_ALLOC();
*driver = (StorageWriteS3)
*this = (StorageWriteS3)
{
.storage = storage,
.partSize = partSize,
@ -307,10 +307,8 @@ storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
},
},
};
this = storageWriteNew(driver, &driver->interface);
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
}

74
src/storage/storage.c
View File

@ -21,7 +21,6 @@ Object type
struct Storage
{
StoragePub pub; // Publicly accessible variables
MemContext *memContext;
const String *path;
mode_t modeFile;
mode_t modePath;
@ -32,8 +31,8 @@ struct Storage
/**********************************************************************************************************************************/
FN_EXTERN Storage *
storageNew(
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface)
const StringId type, const String *const path, const mode_t modeFile, const mode_t modePath, const bool write,
StoragePathExpressionCallback pathExpressionFunction, void *const driver, const StorageInterface interface)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM(STRING_ID, type);
@ -58,45 +57,50 @@ storageNew(
ASSERT(interface.pathRemove != NULL);
ASSERT(interface.remove != NULL);
Storage *this = (Storage *)memNew(sizeof(Storage));
Storage *this = NULL;
*this = (Storage)
OBJ_NEW_BEGIN(Storage, .childQty = MEM_CONTEXT_QTY_MAX)
{
.pub =
this = OBJ_NEW_ALLOC();
*this = (Storage)
{
.type = type,
.driver = driver,
.interface = interface,
},
.memContext = memContextCurrent(),
.path = strDup(path),
.modeFile = modeFile,
.modePath = modePath,
.write = write,
.pathExpressionFunction = pathExpressionFunction,
};
.pub =
{
.type = type,
.driver = objMove(driver, objMemContext(this)),
.interface = interface,
},
.path = strDup(path),
.modeFile = modeFile,
.modePath = modePath,
.write = write,
.pathExpressionFunction = pathExpressionFunction,
};
// If path sync feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath),
"path feature required");
// If path sync feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath),
"path feature required");
// If hardlink feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath),
"path feature required");
// If hardlink feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath),
"path feature required");
// If symlink feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath),
"path feature required");
// If symlink feature is enabled then path feature must be enabled
CHECK(
AssertError, !storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath),
"path feature required");
// If link features are enabled then linkCreate must be implemented
CHECK(
AssertError,
(!storageFeature(this, storageFeatureSymLink) && !storageFeature(this, storageFeatureHardLink)) ||
interface.linkCreate != NULL,
"linkCreate required");
// If link features are enabled then linkCreate must be implemented
CHECK(
AssertError,
(!storageFeature(this, storageFeatureSymLink) && !storageFeature(this, storageFeatureHardLink)) ||
interface.linkCreate != NULL,
"linkCreate required");
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE, this);
}

27
src/storage/write.c
View File

@ -14,7 +14,7 @@ Object type
struct StorageWrite
{
StorageWritePub pub; // Publicly accessible variables
void *driver;
void *driver; // Driver
};
/***********************************************************************************************************************************
@ -30,7 +30,7 @@ This object expects its context to be created in advance. This is so the callin
required multiple functions and contexts to make it safe.
***********************************************************************************************************************************/
FN_EXTERN StorageWrite *
storageWriteNew(void *driver, const StorageWriteInterface *interface)
storageWriteNew(void *const driver, const StorageWriteInterface *const interface)
{
FUNCTION_LOG_BEGIN(logLevelTrace);
FUNCTION_LOG_PARAM_P(VOID, driver);
@ -42,18 +42,23 @@ storageWriteNew(void *driver, const StorageWriteInterface *interface)
ASSERT(driver != NULL);
ASSERT(interface != NULL);
StorageWrite *this = memNew(sizeof(StorageWrite));
StorageWrite *this = NULL;
*this = (StorageWrite)
OBJ_NEW_BEGIN(StorageWrite, .childQty = MEM_CONTEXT_QTY_MAX)
{
.pub =
this = OBJ_NEW_ALLOC();
*this = (StorageWrite)
{
.memContext = memContextCurrent(),
.interface = interface,
.io = ioWriteNew(driver, interface->ioInterface),
},
.driver = driver,
};
.pub =
{
.interface = interface,
.io = ioWriteNew(driver, interface->ioInterface),
},
.driver = objMove(driver, objMemContext(this)),
};
}
OBJ_NEW_END();
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
}

5
src/storage/write.h
View File

@ -23,7 +23,7 @@ Functions
FN_INLINE_ALWAYS StorageWrite *
storageWriteMove(StorageWrite *const this, MemContext *const parentNew)
{
return objMoveContext(this, parentNew);
return objMove(this, parentNew);
}
/***********************************************************************************************************************************
@ -31,7 +31,6 @@ Getters/Setters
***********************************************************************************************************************************/
typedef struct StorageWritePub
{
MemContext *memContext; // Mem context
const StorageWriteInterface *interface; // File data (name, driver type, etc.)
IoWrite *io; // Write interface
} StorageWritePub;
@ -113,7 +112,7 @@ Destructor
FN_INLINE_ALWAYS void
storageWriteFree(StorageWrite *const this)
{
objFreeContext(this);
objFree(this);
}
/***********************************************************************************************************************************

20
test/src/module/performance/storageTest.c
View File

@ -145,15 +145,23 @@ testRun(void)
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleRemote);
// Create a driver to test remote performance of storageNewItrP() and inject it into storageRepo()
StorageTestPerfList driver =
StorageTestPerfList *driver = NULL;
OBJ_NEW_BEGIN(StorageTestPerfList, .childQty = MEM_CONTEXT_QTY_MAX)
{
.interface = storageInterfaceTestDummy,
.fileTotal = fileTotal,
};
driver = OBJ_NEW_ALLOC();
driver.interface.list = storageTestPerfList;
*driver = (StorageTestPerfList)
{
.interface = storageInterfaceTestDummy,
.fileTotal = fileTotal,
};
}
OBJ_NEW_END();
Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
driver->interface.list = storageTestPerfList;
Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, driver, driver->interface);
storageHelper.storageRepoWrite = memNew(sizeof(Storage *));
storageHelper.storageRepoWrite[0] = storageTest;

28
test/src/module/performance/typeTest.c
View File

@ -249,17 +249,25 @@ testRun(void)
ASSERT(TEST_SCALE <= 1000000);
// Create a storage driver to test manifest build with an arbitrary number of files
StorageTestManifestNewBuild driver =
{
.interface = storageInterfaceTestDummy,
.fileTotal = 100000 * (unsigned int)TEST_SCALE,
};
StorageTestManifestNewBuild *driver = NULL;
driver.interface.info = storageTestManifestNewBuildInfo;
driver.interface.list = storageTestManifestNewBuildList;
OBJ_NEW_BEGIN(StorageTestManifestNewBuild, .childQty = MEM_CONTEXT_QTY_MAX)
{
driver = OBJ_NEW_ALLOC();
*driver = (StorageTestManifestNewBuild)
{
.interface = storageInterfaceTestDummy,
.fileTotal = 100000 * (unsigned int)TEST_SCALE,
};
}
OBJ_NEW_END();
driver->interface.info = storageTestManifestNewBuildInfo;
driver->interface.list = storageTestManifestNewBuildList;
const Storage *const storagePg = storageNew(
strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, driver, driver->interface);
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("build manifest");
@ -280,7 +288,7 @@ testRun(void)
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, " check file total");
TEST_RESULT_UINT(manifestFileTotal(manifest), driver->fileTotal, " check file total");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("save manifest");
@ -310,7 +318,7 @@ testRun(void)
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, " check file total");
TEST_RESULT_UINT(manifestFileTotal(manifest), driver->fileTotal, " check file total");
// -------------------------------------------------------------------------------------------------------------------------
TEST_TITLE("find all files");