You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-03 00:26:59 +02:00
Improve interface handling in storage module.
Make the interface object the parent of the driver object rather than the interface being allocated directly in the driver object. The prior method was more efficient when mem contexts had a much higher cost. Now mem contexts are cheap so it makes more sense to structure the objects in a way that works better with mem context auditing. This also means the mem context does not need to be stored separately since it can be extracted directly from the interface object. There are other areas that need to get the same improvement before the specialized objMoveContext() and objFreeContext() functions can be removed.
This commit is contained in:
@ -123,13 +123,13 @@ storageReadAzureNew(
|
|||||||
ASSERT(storage != NULL);
|
ASSERT(storage != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageRead *this = NULL;
|
StorageReadAzure *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageReadAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageReadAzure, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageReadAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadAzure);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageReadAzure)
|
*this = (StorageReadAzure)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
|
|
||||||
@ -149,10 +149,8 @@ storageReadAzureNew(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageReadNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -734,13 +734,13 @@ storageAzureNew(
|
|||||||
ASSERT(key != NULL);
|
ASSERT(key != NULL);
|
||||||
ASSERT(blockSize != 0);
|
ASSERT(blockSize != 0);
|
||||||
|
|
||||||
Storage *this = NULL;
|
StorageAzure *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageAzure, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageAzure);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageAzure)
|
*this = (StorageAzure)
|
||||||
{
|
{
|
||||||
.interface = storageInterfaceAzure,
|
.interface = storageInterfaceAzure,
|
||||||
.container = strDup(container),
|
.container = strDup(container),
|
||||||
@ -754,32 +754,30 @@ storageAzureNew(
|
|||||||
|
|
||||||
// Store shared key or parse sas query
|
// Store shared key or parse sas query
|
||||||
if (keyType == storageAzureKeyTypeShared)
|
if (keyType == storageAzureKeyTypeShared)
|
||||||
driver->sharedKey = bufNewDecode(encodingBase64, key);
|
this->sharedKey = bufNewDecode(encodingBase64, key);
|
||||||
else
|
else
|
||||||
driver->sasKey = httpQueryNewStr(key);
|
this->sasKey = httpQueryNewStr(key);
|
||||||
|
|
||||||
// Create the http client used to service requests
|
// Create the http client used to service requests
|
||||||
driver->httpClient = httpClientNew(
|
this->httpClient = httpClientNew(
|
||||||
tlsClientNewP(
|
tlsClientNewP(
|
||||||
sckClientNew(driver->host, port, timeout, timeout), driver->host, timeout, timeout, verifyPeer, .caFile = caFile,
|
sckClientNew(this->host, port, timeout, timeout), this->host, timeout, timeout, verifyPeer, .caFile = caFile,
|
||||||
.caPath = caPath),
|
.caPath = caPath),
|
||||||
timeout);
|
timeout);
|
||||||
|
|
||||||
// Create list of redacted headers
|
// Create list of redacted headers
|
||||||
driver->headerRedactList = strLstNew();
|
this->headerRedactList = strLstNew();
|
||||||
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
||||||
strLstAdd(driver->headerRedactList, HTTP_HEADER_DATE_STR);
|
strLstAdd(this->headerRedactList, HTTP_HEADER_DATE_STR);
|
||||||
|
|
||||||
// Create list of redacted query keys
|
// Create list of redacted query keys
|
||||||
driver->queryRedactList = strLstNew();
|
this->queryRedactList = strLstNew();
|
||||||
strLstAdd(driver->queryRedactList, AZURE_QUERY_SIG_STR);
|
strLstAdd(this->queryRedactList, AZURE_QUERY_SIG_STR);
|
||||||
|
|
||||||
// Generate starting file id
|
// Generate starting file id
|
||||||
cryptoRandomBytes((unsigned char *)&driver->fileId, sizeof(driver->fileId));
|
cryptoRandomBytes((unsigned char *)&this->fileId, sizeof(this->fileId));
|
||||||
|
|
||||||
this = storageNew(STORAGE_AZURE_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_AZURE_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
|
||||||
}
|
}
|
||||||
|
@ -262,7 +262,7 @@ storageWriteAzureClose(THIS_VOID)
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
FN_EXTERN StorageWrite *
|
FN_EXTERN StorageWrite *
|
||||||
storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId, size_t blockSize)
|
storageWriteAzureNew(StorageAzure *const storage, const String *const name, const uint64_t fileId, const size_t blockSize)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(STORAGE_AZURE, storage);
|
FUNCTION_LOG_PARAM(STORAGE_AZURE, storage);
|
||||||
@ -274,13 +274,13 @@ storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId,
|
|||||||
ASSERT(storage != NULL);
|
ASSERT(storage != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageWrite *this = NULL;
|
StorageWriteAzure *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageWriteAzure, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageWriteAzure, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageWriteAzure *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteAzure);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageWriteAzure)
|
*this = (StorageWriteAzure)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
.fileId = fileId,
|
.fileId = fileId,
|
||||||
@ -304,10 +304,8 @@ storageWriteAzureNew(StorageAzure *storage, const String *name, uint64_t fileId,
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageWriteNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -130,13 +130,13 @@ storageReadGcsNew(
|
|||||||
ASSERT(storage != NULL);
|
ASSERT(storage != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageRead *this = NULL;
|
StorageReadGcs *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageReadGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageReadGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageReadGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadGcs);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageReadGcs)
|
*this = (StorageReadGcs)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
|
|
||||||
@ -156,10 +156,8 @@ storageReadGcsNew(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageReadNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -946,9 +946,9 @@ static const StorageInterface storageInterfaceGcs =
|
|||||||
|
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
storageGcsNew(
|
storageGcsNew(
|
||||||
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
|
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket,
|
||||||
StorageGcsKeyType keyType, const String *key, size_t chunkSize, const String *endpoint, TimeMSec timeout, bool verifyPeer,
|
const StorageGcsKeyType keyType, const String *const key, const size_t chunkSize, const String *const endpoint,
|
||||||
const String *caFile, const String *caPath)
|
const TimeMSec timeout, const bool verifyPeer, const String *const caFile, const String *const caPath)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(STRING, path);
|
FUNCTION_LOG_PARAM(STRING, path);
|
||||||
@ -970,13 +970,13 @@ storageGcsNew(
|
|||||||
ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL);
|
ASSERT(keyType == storageGcsKeyTypeAuto || key != NULL);
|
||||||
ASSERT(chunkSize != 0);
|
ASSERT(chunkSize != 0);
|
||||||
|
|
||||||
Storage *this = NULL;
|
StorageGcs *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageGcs);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageGcs)
|
*this = (StorageGcs)
|
||||||
{
|
{
|
||||||
.interface = storageInterfaceGcs,
|
.interface = storageInterfaceGcs,
|
||||||
.write = write,
|
.write = write,
|
||||||
@ -991,11 +991,11 @@ storageGcsNew(
|
|||||||
// Auto authentication for GCE instances
|
// Auto authentication for GCE instances
|
||||||
case storageGcsKeyTypeAuto:
|
case storageGcsKeyTypeAuto:
|
||||||
{
|
{
|
||||||
driver->authUrl = httpUrlNewParseP(
|
this->authUrl = httpUrlNewParseP(
|
||||||
STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
|
STRDEF("metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"),
|
||||||
.type = httpProtocolTypeHttp);
|
.type = httpProtocolTypeHttp);
|
||||||
driver->authClient = httpClientNew(
|
this->authClient = httpClientNew(
|
||||||
sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout, timeout), timeout);
|
sckClientNew(httpUrlHost(this->authUrl), httpUrlPort(this->authUrl), timeout, timeout), timeout);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -1004,18 +1004,18 @@ storageGcsNew(
|
|||||||
case storageGcsKeyTypeService:
|
case storageGcsKeyTypeService:
|
||||||
{
|
{
|
||||||
KeyValue *kvKey = varKv(jsonToVar(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key)))));
|
KeyValue *kvKey = varKv(jsonToVar(strNewBuf(storageGetP(storageNewReadP(storagePosixNewP(FSLASH_STR), key)))));
|
||||||
driver->credential = varStr(kvGet(kvKey, GCS_JSON_CLIENT_EMAIL_VAR));
|
this->credential = varStr(kvGet(kvKey, GCS_JSON_CLIENT_EMAIL_VAR));
|
||||||
driver->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR));
|
this->privateKey = varStr(kvGet(kvKey, GCS_JSON_PRIVATE_KEY_VAR));
|
||||||
const String *const uri = varStr(kvGet(kvKey, GCS_JSON_TOKEN_URI_VAR));
|
const String *const uri = varStr(kvGet(kvKey, GCS_JSON_TOKEN_URI_VAR));
|
||||||
|
|
||||||
CHECK(FormatError, driver->credential != NULL && driver->privateKey != NULL && uri != NULL, "credentials missing");
|
CHECK(FormatError, this->credential != NULL && this->privateKey != NULL && uri != NULL, "credentials missing");
|
||||||
|
|
||||||
driver->authUrl = httpUrlNewParseP(uri, .type = httpProtocolTypeHttps);
|
this->authUrl = httpUrlNewParseP(uri, .type = httpProtocolTypeHttps);
|
||||||
|
|
||||||
driver->authClient = httpClientNew(
|
this->authClient = httpClientNew(
|
||||||
tlsClientNewP(
|
tlsClientNewP(
|
||||||
sckClientNew(httpUrlHost(driver->authUrl), httpUrlPort(driver->authUrl), timeout, timeout),
|
sckClientNew(httpUrlHost(this->authUrl), httpUrlPort(this->authUrl), timeout, timeout),
|
||||||
httpUrlHost(driver->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
|
httpUrlHost(this->authUrl), timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
|
||||||
timeout);
|
timeout);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
@ -1023,33 +1023,31 @@ storageGcsNew(
|
|||||||
|
|
||||||
// Store the authentication token
|
// Store the authentication token
|
||||||
case storageGcsKeyTypeToken:
|
case storageGcsKeyTypeToken:
|
||||||
driver->token = strDup(key);
|
this->token = strDup(key);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Parse the endpoint to extract the host and port
|
// Parse the endpoint to extract the host and port
|
||||||
HttpUrl *url = httpUrlNewParseP(endpoint, .type = httpProtocolTypeHttps);
|
HttpUrl *url = httpUrlNewParseP(endpoint, .type = httpProtocolTypeHttps);
|
||||||
driver->endpoint = httpUrlHost(url);
|
this->endpoint = httpUrlHost(url);
|
||||||
|
|
||||||
// Create the http client used to service requests
|
// Create the http client used to service requests
|
||||||
driver->httpClient = httpClientNew(
|
this->httpClient = httpClientNew(
|
||||||
tlsClientNewP(
|
tlsClientNewP(
|
||||||
sckClientNew(driver->endpoint, httpUrlPort(url), timeout, timeout), driver->endpoint, timeout, timeout, verifyPeer,
|
sckClientNew(this->endpoint, httpUrlPort(url), timeout, timeout), this->endpoint, timeout, timeout, verifyPeer,
|
||||||
.caFile = caFile, .caPath = caPath),
|
.caFile = caFile, .caPath = caPath),
|
||||||
timeout);
|
timeout);
|
||||||
|
|
||||||
// Create list of redacted headers
|
// Create list of redacted headers
|
||||||
driver->headerRedactList = strLstNew();
|
this->headerRedactList = strLstNew();
|
||||||
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
||||||
strLstAdd(driver->headerRedactList, GCS_HEADER_UPLOAD_ID_STR);
|
strLstAdd(this->headerRedactList, GCS_HEADER_UPLOAD_ID_STR);
|
||||||
|
|
||||||
// Create list of redacted query keys
|
// Create list of redacted query keys
|
||||||
driver->queryRedactList = strLstNew();
|
this->queryRedactList = strLstNew();
|
||||||
strLstAdd(driver->queryRedactList, GCS_QUERY_UPLOAD_ID_STR);
|
strLstAdd(this->queryRedactList, GCS_QUERY_UPLOAD_ID_STR);
|
||||||
|
|
||||||
this = storageNew(STORAGE_GCS_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_GCS_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
|
||||||
}
|
}
|
||||||
|
@ -321,7 +321,7 @@ storageWriteGcsClose(THIS_VOID)
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
FN_EXTERN StorageWrite *
|
FN_EXTERN StorageWrite *
|
||||||
storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
|
storageWriteGcsNew(StorageGcs *const storage, const String *const name, const size_t chunkSize)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(STORAGE_GCS, storage);
|
FUNCTION_LOG_PARAM(STORAGE_GCS, storage);
|
||||||
@ -332,13 +332,13 @@ storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
|
|||||||
ASSERT(storage != NULL);
|
ASSERT(storage != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageWrite *this = NULL;
|
StorageWriteGcs *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageWriteGcs, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageWriteGcs, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageWriteGcs *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteGcs);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageWriteGcs)
|
*this = (StorageWriteGcs)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
.chunkSize = chunkSize,
|
.chunkSize = chunkSize,
|
||||||
@ -361,10 +361,8 @@ storageWriteGcsNew(StorageGcs *storage, const String *name, size_t chunkSize)
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageWriteNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -220,13 +220,13 @@ storageReadPosixNew(
|
|||||||
|
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageRead *this = NULL;
|
StorageReadPosix *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageReadPosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
|
OBJ_NEW_BEGIN(StorageReadPosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
|
||||||
{
|
{
|
||||||
StorageReadPosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadPosix);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageReadPosix)
|
*this = (StorageReadPosix)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
.fd = -1,
|
.fd = -1,
|
||||||
@ -254,10 +254,8 @@ storageReadPosixNew(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageReadNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -590,8 +590,8 @@ static const StorageInterface storageInterfacePosix =
|
|||||||
|
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
storagePosixNewInternal(
|
storagePosixNewInternal(
|
||||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
const StringId type, const String *const path, const mode_t modeFile, const mode_t modePath, const bool write,
|
||||||
StoragePathExpressionCallback pathExpressionFunction, bool pathSync)
|
StoragePathExpressionCallback pathExpressionFunction, const bool pathSync)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(STRING_ID, type);
|
FUNCTION_LOG_PARAM(STRING_ID, type);
|
||||||
@ -612,32 +612,30 @@ storagePosixNewInternal(
|
|||||||
userInit();
|
userInit();
|
||||||
|
|
||||||
// Create the object
|
// Create the object
|
||||||
Storage *this = NULL;
|
StoragePosix *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StoragePosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StoragePosix, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StoragePosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StoragePosix);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StoragePosix)
|
*this = (StoragePosix)
|
||||||
{
|
{
|
||||||
.interface = storageInterfacePosix,
|
.interface = storageInterfacePosix,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Disable path sync when not supported
|
// Disable path sync when not supported
|
||||||
if (!pathSync)
|
if (!pathSync)
|
||||||
driver->interface.pathSync = NULL;
|
this->interface.pathSync = NULL;
|
||||||
|
|
||||||
// If this is a posix driver then add link features
|
// If this is a posix driver then add link features
|
||||||
if (type == STORAGE_POSIX_TYPE)
|
if (type == STORAGE_POSIX_TYPE)
|
||||||
driver->interface.feature |=
|
this->interface.feature |=
|
||||||
1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync |
|
1 << storageFeatureHardLink | 1 << storageFeatureSymLink | 1 << storageFeaturePathSync |
|
||||||
1 << storageFeatureInfoDetail;
|
1 << storageFeatureInfoDetail;
|
||||||
|
|
||||||
this = storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(STORAGE, storageNew(type, path, modeFile, modePath, write, pathExpressionFunction, this, this->interface));
|
||||||
}
|
}
|
||||||
|
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
|
@ -245,13 +245,13 @@ storageWritePosixNew(
|
|||||||
ASSERT(modeFile != 0);
|
ASSERT(modeFile != 0);
|
||||||
ASSERT(modePath != 0);
|
ASSERT(modePath != 0);
|
||||||
|
|
||||||
StorageWrite *this = NULL;
|
StorageWritePosix *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageWritePosix, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
|
OBJ_NEW_BEGIN(StorageWritePosix, .childQty = MEM_CONTEXT_QTY_MAX, .callbackQty = 1)
|
||||||
{
|
{
|
||||||
StorageWritePosix *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWritePosix);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageWritePosix)
|
*this = (StorageWritePosix)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
.path = strPath(name),
|
.path = strPath(name),
|
||||||
@ -283,11 +283,9 @@ storageWritePosixNew(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Create temp file name
|
// Create temp file name
|
||||||
driver->nameTmp = atomic ? strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strZ(name)) : driver->interface.name;
|
this->nameTmp = atomic ? strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strZ(name)) : this->interface.name;
|
||||||
|
|
||||||
this = storageWriteNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,7 @@ Object type
|
|||||||
struct StorageRead
|
struct StorageRead
|
||||||
{
|
{
|
||||||
StorageReadPub pub; // Publicly accessible variables
|
StorageReadPub pub; // Publicly accessible variables
|
||||||
void *driver;
|
void *driver; // Driver
|
||||||
};
|
};
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -27,7 +27,7 @@ Macros for function logging
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
FN_EXTERN StorageRead *
|
FN_EXTERN StorageRead *
|
||||||
storageReadNew(void *driver, const StorageReadInterface *interface)
|
storageReadNew(void *const driver, const StorageReadInterface *const interface)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM_P(VOID, driver);
|
FUNCTION_LOG_PARAM_P(VOID, driver);
|
||||||
@ -41,18 +41,21 @@ storageReadNew(void *driver, const StorageReadInterface *interface)
|
|||||||
|
|
||||||
StorageRead *this = NULL;
|
StorageRead *this = NULL;
|
||||||
|
|
||||||
this = memNew(sizeof(StorageRead));
|
OBJ_NEW_BEGIN(StorageRead, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
|
|
||||||
*this = (StorageRead)
|
|
||||||
{
|
{
|
||||||
.pub =
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
|
*this = (StorageRead)
|
||||||
{
|
{
|
||||||
.memContext = memContextCurrent(),
|
.pub =
|
||||||
.interface = interface,
|
{
|
||||||
.io = ioReadNew(driver, interface->ioInterface),
|
.interface = interface,
|
||||||
},
|
.io = ioReadNew(driver, interface->ioInterface),
|
||||||
.driver = driver,
|
},
|
||||||
};
|
.driver = objMove(driver, objMemContext(this)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
||||||
}
|
}
|
||||||
|
@ -20,7 +20,7 @@ Functions
|
|||||||
FN_INLINE_ALWAYS StorageRead *
|
FN_INLINE_ALWAYS StorageRead *
|
||||||
storageReadMove(StorageRead *const this, MemContext *const parentNew)
|
storageReadMove(StorageRead *const this, MemContext *const parentNew)
|
||||||
{
|
{
|
||||||
return objMoveContext(this, parentNew);
|
return objMove(this, parentNew);
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -28,7 +28,6 @@ Getters/Setters
|
|||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
typedef struct StorageReadPub
|
typedef struct StorageReadPub
|
||||||
{
|
{
|
||||||
MemContext *memContext; // Mem context
|
|
||||||
const StorageReadInterface *interface; // File data (name, driver type, etc.)
|
const StorageReadInterface *interface; // File data (name, driver type, etc.)
|
||||||
IoRead *io; // Read interface
|
IoRead *io; // Read interface
|
||||||
} StorageReadPub;
|
} StorageReadPub;
|
||||||
@ -81,7 +80,7 @@ Destructor
|
|||||||
FN_INLINE_ALWAYS void
|
FN_INLINE_ALWAYS void
|
||||||
storageReadFree(StorageRead *const this)
|
storageReadFree(StorageRead *const this)
|
||||||
{
|
{
|
||||||
objFreeContext(this);
|
objFree(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
|
@ -470,8 +470,8 @@ static const StorageInterface storageInterfaceRemote =
|
|||||||
|
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
storageRemoteNew(
|
storageRemoteNew(
|
||||||
mode_t modeFile, mode_t modePath, bool write, StoragePathExpressionCallback pathExpressionFunction, ProtocolClient *client,
|
const mode_t modeFile, const mode_t modePath, const bool write, StoragePathExpressionCallback pathExpressionFunction,
|
||||||
unsigned int compressLevel)
|
ProtocolClient *const client, const unsigned int compressLevel)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(MODE, modeFile);
|
FUNCTION_LOG_PARAM(MODE, modeFile);
|
||||||
@ -486,26 +486,25 @@ storageRemoteNew(
|
|||||||
ASSERT(modePath != 0);
|
ASSERT(modePath != 0);
|
||||||
ASSERT(client != NULL);
|
ASSERT(client != NULL);
|
||||||
|
|
||||||
Storage *this = NULL;
|
StorageRemote *this = NULL;
|
||||||
|
const String *path = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageRemote, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageRemote, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageRemote *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageRemote);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageRemote)
|
*this = (StorageRemote)
|
||||||
{
|
{
|
||||||
.client = client,
|
.client = client,
|
||||||
.compressLevel = compressLevel,
|
.compressLevel = compressLevel,
|
||||||
.interface = storageInterfaceRemote,
|
.interface = storageInterfaceRemote,
|
||||||
};
|
};
|
||||||
|
|
||||||
const String *path = NULL;
|
|
||||||
|
|
||||||
// Get storage features from the remote
|
// Get storage features from the remote
|
||||||
MEM_CONTEXT_TEMP_BEGIN()
|
MEM_CONTEXT_TEMP_BEGIN()
|
||||||
{
|
{
|
||||||
// Execute command and get result
|
// Execute command and get result
|
||||||
PackRead *result = protocolClientExecute(driver->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
|
PackRead *result = protocolClientExecute(this->client, protocolCommandNew(PROTOCOL_COMMAND_STORAGE_FEATURE), true);
|
||||||
|
|
||||||
// Get path in parent context
|
// Get path in parent context
|
||||||
MEM_CONTEXT_PRIOR_BEGIN()
|
MEM_CONTEXT_PRIOR_BEGIN()
|
||||||
@ -514,13 +513,12 @@ storageRemoteNew(
|
|||||||
}
|
}
|
||||||
MEM_CONTEXT_PRIOR_END();
|
MEM_CONTEXT_PRIOR_END();
|
||||||
|
|
||||||
driver->interface.feature = pckReadU64P(result);
|
this->interface.feature = pckReadU64P(result);
|
||||||
}
|
}
|
||||||
MEM_CONTEXT_TEMP_END();
|
MEM_CONTEXT_TEMP_END();
|
||||||
|
|
||||||
this = storageNew(STORAGE_REMOTE_TYPE, path, modeFile, modePath, write, pathExpressionFunction, driver, driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(
|
||||||
|
STORAGE, storageNew(STORAGE_REMOTE_TYPE, path, modeFile, modePath, write, pathExpressionFunction, this, this->interface));
|
||||||
}
|
}
|
||||||
|
@ -126,13 +126,13 @@ storageReadS3New(
|
|||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
ASSERT(limit == NULL || varUInt64(limit) > 0);
|
ASSERT(limit == NULL || varUInt64(limit) > 0);
|
||||||
|
|
||||||
StorageRead *this = NULL;
|
StorageReadS3 *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageReadS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageReadS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageReadS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageRead::StorageReadS3);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageReadS3)
|
*this = (StorageReadS3)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
|
|
||||||
@ -152,10 +152,8 @@ storageReadS3New(
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageReadNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_READ, this);
|
FUNCTION_LOG_RETURN(STORAGE_READ, storageReadNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -1093,11 +1093,12 @@ static const StorageInterface storageInterfaceS3 =
|
|||||||
|
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
storageS3New(
|
storageS3New(
|
||||||
const String *path, bool write, StoragePathExpressionCallback pathExpressionFunction, const String *bucket,
|
const String *const path, const bool write, StoragePathExpressionCallback pathExpressionFunction, const String *const bucket,
|
||||||
const String *endPoint, StorageS3UriStyle uriStyle, const String *region, StorageS3KeyType keyType, const String *accessKey,
|
const String *const endPoint, const StorageS3UriStyle uriStyle, const String *const region, const StorageS3KeyType keyType,
|
||||||
const String *secretAccessKey, const String *securityToken, const String *const kmsKeyId, const String *credRole,
|
const String *const accessKey, const String *const secretAccessKey, const String *const securityToken,
|
||||||
const String *const webIdToken, size_t partSize, const String *host, unsigned int port, TimeMSec timeout, bool verifyPeer,
|
const String *const kmsKeyId, const String *const credRole, const String *const webIdToken, const size_t partSize,
|
||||||
const String *caFile, const String *caPath)
|
const String *host, const unsigned int port, const TimeMSec timeout, const bool verifyPeer, const String *const caFile,
|
||||||
|
const String *const caPath)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelDebug);
|
FUNCTION_LOG_BEGIN(logLevelDebug);
|
||||||
FUNCTION_LOG_PARAM(STRING, path);
|
FUNCTION_LOG_PARAM(STRING, path);
|
||||||
@ -1129,13 +1130,13 @@ storageS3New(
|
|||||||
ASSERT(region != NULL);
|
ASSERT(region != NULL);
|
||||||
ASSERT(partSize != 0);
|
ASSERT(partSize != 0);
|
||||||
|
|
||||||
Storage *this = NULL;
|
StorageS3 *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageS3, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), Storage::StorageS3);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageS3)
|
*this = (StorageS3)
|
||||||
{
|
{
|
||||||
.interface = storageInterfaceS3,
|
.interface = storageInterfaceS3,
|
||||||
.bucket = strDup(bucket),
|
.bucket = strDup(bucket),
|
||||||
@ -1154,26 +1155,26 @@ storageS3New(
|
|||||||
|
|
||||||
// Create the HTTP client used to service requests
|
// Create the HTTP client used to service requests
|
||||||
if (host == NULL)
|
if (host == NULL)
|
||||||
host = driver->bucketEndpoint;
|
host = this->bucketEndpoint;
|
||||||
|
|
||||||
driver->httpClient = httpClientNew(
|
this->httpClient = httpClientNew(
|
||||||
tlsClientNewP(
|
tlsClientNewP(
|
||||||
sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
|
sckClientNew(host, port, timeout, timeout), host, timeout, timeout, verifyPeer, .caFile = caFile, .caPath = caPath),
|
||||||
timeout);
|
timeout);
|
||||||
|
|
||||||
// Initialize authentication
|
// Initialize authentication
|
||||||
switch (driver->keyType)
|
switch (this->keyType)
|
||||||
{
|
{
|
||||||
// Create the HTTP client used to retrieve temporary security credentials
|
// Create the HTTP client used to retrieve temporary security credentials
|
||||||
case storageS3KeyTypeAuto:
|
case storageS3KeyTypeAuto:
|
||||||
{
|
{
|
||||||
ASSERT(accessKey == NULL && secretAccessKey == NULL && securityToken == NULL);
|
ASSERT(accessKey == NULL && secretAccessKey == NULL && securityToken == NULL);
|
||||||
|
|
||||||
driver->credRole = strDup(credRole);
|
this->credRole = strDup(credRole);
|
||||||
driver->credHost = S3_CREDENTIAL_HOST_STR;
|
this->credHost = S3_CREDENTIAL_HOST_STR;
|
||||||
driver->credExpirationTime = time(NULL);
|
this->credExpirationTime = time(NULL);
|
||||||
driver->credHttpClient = httpClientNew(
|
this->credHttpClient = httpClientNew(
|
||||||
sckClientNew(driver->credHost, S3_CREDENTIAL_PORT, timeout, timeout), timeout);
|
sckClientNew(this->credHost, S3_CREDENTIAL_PORT, timeout, timeout), timeout);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
@ -1185,13 +1186,13 @@ storageS3New(
|
|||||||
ASSERT(credRole != NULL);
|
ASSERT(credRole != NULL);
|
||||||
ASSERT(webIdToken != NULL);
|
ASSERT(webIdToken != NULL);
|
||||||
|
|
||||||
driver->credRole = strDup(credRole);
|
this->credRole = strDup(credRole);
|
||||||
driver->webIdToken = strDup(webIdToken);
|
this->webIdToken = strDup(webIdToken);
|
||||||
driver->credHost = S3_STS_HOST_STR;
|
this->credHost = S3_STS_HOST_STR;
|
||||||
driver->credExpirationTime = time(NULL);
|
this->credExpirationTime = time(NULL);
|
||||||
driver->credHttpClient = httpClientNew(
|
this->credHttpClient = httpClientNew(
|
||||||
tlsClientNewP(
|
tlsClientNewP(
|
||||||
sckClientNew(driver->credHost, S3_STS_PORT, timeout, timeout), driver->credHost, timeout, timeout, true,
|
sckClientNew(this->credHost, S3_STS_PORT, timeout, timeout), this->credHost, timeout, timeout, true,
|
||||||
.caFile = caFile, .caPath = caPath),
|
.caFile = caFile, .caPath = caPath),
|
||||||
timeout);
|
timeout);
|
||||||
|
|
||||||
@ -1201,26 +1202,24 @@ storageS3New(
|
|||||||
// Set shared key credentials
|
// Set shared key credentials
|
||||||
default:
|
default:
|
||||||
{
|
{
|
||||||
ASSERT(driver->keyType == storageS3KeyTypeShared);
|
ASSERT(this->keyType == storageS3KeyTypeShared);
|
||||||
ASSERT(accessKey != NULL && secretAccessKey != NULL);
|
ASSERT(accessKey != NULL && secretAccessKey != NULL);
|
||||||
|
|
||||||
driver->accessKey = strDup(accessKey);
|
this->accessKey = strDup(accessKey);
|
||||||
driver->secretAccessKey = strDup(secretAccessKey);
|
this->secretAccessKey = strDup(secretAccessKey);
|
||||||
driver->securityToken = strDup(securityToken);
|
this->securityToken = strDup(securityToken);
|
||||||
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create list of redacted headers
|
// Create list of redacted headers
|
||||||
driver->headerRedactList = strLstNew();
|
this->headerRedactList = strLstNew();
|
||||||
strLstAdd(driver->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
strLstAdd(this->headerRedactList, HTTP_HEADER_AUTHORIZATION_STR);
|
||||||
strLstAdd(driver->headerRedactList, S3_HEADER_DATE_STR);
|
strLstAdd(this->headerRedactList, S3_HEADER_DATE_STR);
|
||||||
strLstAdd(driver->headerRedactList, S3_HEADER_TOKEN_STR);
|
strLstAdd(this->headerRedactList, S3_HEADER_TOKEN_STR);
|
||||||
|
|
||||||
this = storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, driver, driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(STORAGE, storageNew(STORAGE_S3_TYPE, path, 0, 0, write, pathExpressionFunction, this, this->interface));
|
||||||
}
|
}
|
||||||
|
@ -268,7 +268,7 @@ storageWriteS3Close(THIS_VOID)
|
|||||||
|
|
||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
FN_EXTERN StorageWrite *
|
FN_EXTERN StorageWrite *
|
||||||
storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
|
storageWriteS3New(StorageS3 *const storage, const String *const name, const size_t partSize)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(STORAGE_S3, storage);
|
FUNCTION_LOG_PARAM(STORAGE_S3, storage);
|
||||||
@ -278,13 +278,13 @@ storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
|
|||||||
ASSERT(storage != NULL);
|
ASSERT(storage != NULL);
|
||||||
ASSERT(name != NULL);
|
ASSERT(name != NULL);
|
||||||
|
|
||||||
StorageWrite *this = NULL;
|
StorageWriteS3 *this = NULL;
|
||||||
|
|
||||||
OBJ_NEW_BEGIN(StorageWriteS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
OBJ_NEW_BEGIN(StorageWriteS3, .childQty = MEM_CONTEXT_QTY_MAX, .allocQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
StorageWriteS3 *const driver = OBJ_NAME(OBJ_NEW_ALLOC(), StorageWrite::StorageWriteS3);
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
*driver = (StorageWriteS3)
|
*this = (StorageWriteS3)
|
||||||
{
|
{
|
||||||
.storage = storage,
|
.storage = storage,
|
||||||
.partSize = partSize,
|
.partSize = partSize,
|
||||||
@ -307,10 +307,8 @@ storageWriteS3New(StorageS3 *storage, const String *name, size_t partSize)
|
|||||||
},
|
},
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
this = storageWriteNew(driver, &driver->interface);
|
|
||||||
}
|
}
|
||||||
OBJ_NEW_END();
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
FUNCTION_LOG_RETURN(STORAGE_WRITE, storageWriteNew(this, &this->interface));
|
||||||
}
|
}
|
||||||
|
@ -21,7 +21,6 @@ Object type
|
|||||||
struct Storage
|
struct Storage
|
||||||
{
|
{
|
||||||
StoragePub pub; // Publicly accessible variables
|
StoragePub pub; // Publicly accessible variables
|
||||||
MemContext *memContext;
|
|
||||||
const String *path;
|
const String *path;
|
||||||
mode_t modeFile;
|
mode_t modeFile;
|
||||||
mode_t modePath;
|
mode_t modePath;
|
||||||
@ -32,8 +31,8 @@ struct Storage
|
|||||||
/**********************************************************************************************************************************/
|
/**********************************************************************************************************************************/
|
||||||
FN_EXTERN Storage *
|
FN_EXTERN Storage *
|
||||||
storageNew(
|
storageNew(
|
||||||
StringId type, const String *path, mode_t modeFile, mode_t modePath, bool write,
|
const StringId type, const String *const path, const mode_t modeFile, const mode_t modePath, const bool write,
|
||||||
StoragePathExpressionCallback pathExpressionFunction, void *driver, StorageInterface interface)
|
StoragePathExpressionCallback pathExpressionFunction, void *const driver, const StorageInterface interface)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM(STRING_ID, type);
|
FUNCTION_LOG_PARAM(STRING_ID, type);
|
||||||
@ -58,45 +57,50 @@ storageNew(
|
|||||||
ASSERT(interface.pathRemove != NULL);
|
ASSERT(interface.pathRemove != NULL);
|
||||||
ASSERT(interface.remove != NULL);
|
ASSERT(interface.remove != NULL);
|
||||||
|
|
||||||
Storage *this = (Storage *)memNew(sizeof(Storage));
|
Storage *this = NULL;
|
||||||
|
|
||||||
*this = (Storage)
|
OBJ_NEW_BEGIN(Storage, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
.pub =
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
|
*this = (Storage)
|
||||||
{
|
{
|
||||||
.type = type,
|
.pub =
|
||||||
.driver = driver,
|
{
|
||||||
.interface = interface,
|
.type = type,
|
||||||
},
|
.driver = objMove(driver, objMemContext(this)),
|
||||||
.memContext = memContextCurrent(),
|
.interface = interface,
|
||||||
.path = strDup(path),
|
},
|
||||||
.modeFile = modeFile,
|
.path = strDup(path),
|
||||||
.modePath = modePath,
|
.modeFile = modeFile,
|
||||||
.write = write,
|
.modePath = modePath,
|
||||||
.pathExpressionFunction = pathExpressionFunction,
|
.write = write,
|
||||||
};
|
.pathExpressionFunction = pathExpressionFunction,
|
||||||
|
};
|
||||||
|
|
||||||
// If path sync feature is enabled then path feature must be enabled
|
// If path sync feature is enabled then path feature must be enabled
|
||||||
CHECK(
|
CHECK(
|
||||||
AssertError, !storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath),
|
AssertError, !storageFeature(this, storageFeaturePathSync) || storageFeature(this, storageFeaturePath),
|
||||||
"path feature required");
|
"path feature required");
|
||||||
|
|
||||||
// If hardlink feature is enabled then path feature must be enabled
|
// If hardlink feature is enabled then path feature must be enabled
|
||||||
CHECK(
|
CHECK(
|
||||||
AssertError, !storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath),
|
AssertError, !storageFeature(this, storageFeatureHardLink) || storageFeature(this, storageFeaturePath),
|
||||||
"path feature required");
|
"path feature required");
|
||||||
|
|
||||||
// If symlink feature is enabled then path feature must be enabled
|
// If symlink feature is enabled then path feature must be enabled
|
||||||
CHECK(
|
CHECK(
|
||||||
AssertError, !storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath),
|
AssertError, !storageFeature(this, storageFeatureSymLink) || storageFeature(this, storageFeaturePath),
|
||||||
"path feature required");
|
"path feature required");
|
||||||
|
|
||||||
// If link features are enabled then linkCreate must be implemented
|
// If link features are enabled then linkCreate must be implemented
|
||||||
CHECK(
|
CHECK(
|
||||||
AssertError,
|
AssertError,
|
||||||
(!storageFeature(this, storageFeatureSymLink) && !storageFeature(this, storageFeatureHardLink)) ||
|
(!storageFeature(this, storageFeatureSymLink) && !storageFeature(this, storageFeatureHardLink)) ||
|
||||||
interface.linkCreate != NULL,
|
interface.linkCreate != NULL,
|
||||||
"linkCreate required");
|
"linkCreate required");
|
||||||
|
}
|
||||||
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE, this);
|
FUNCTION_LOG_RETURN(STORAGE, this);
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,7 @@ Object type
|
|||||||
struct StorageWrite
|
struct StorageWrite
|
||||||
{
|
{
|
||||||
StorageWritePub pub; // Publicly accessible variables
|
StorageWritePub pub; // Publicly accessible variables
|
||||||
void *driver;
|
void *driver; // Driver
|
||||||
};
|
};
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -30,7 +30,7 @@ This object expects its context to be created in advance. This is so the callin
|
|||||||
required multiple functions and contexts to make it safe.
|
required multiple functions and contexts to make it safe.
|
||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
FN_EXTERN StorageWrite *
|
FN_EXTERN StorageWrite *
|
||||||
storageWriteNew(void *driver, const StorageWriteInterface *interface)
|
storageWriteNew(void *const driver, const StorageWriteInterface *const interface)
|
||||||
{
|
{
|
||||||
FUNCTION_LOG_BEGIN(logLevelTrace);
|
FUNCTION_LOG_BEGIN(logLevelTrace);
|
||||||
FUNCTION_LOG_PARAM_P(VOID, driver);
|
FUNCTION_LOG_PARAM_P(VOID, driver);
|
||||||
@ -42,18 +42,23 @@ storageWriteNew(void *driver, const StorageWriteInterface *interface)
|
|||||||
ASSERT(driver != NULL);
|
ASSERT(driver != NULL);
|
||||||
ASSERT(interface != NULL);
|
ASSERT(interface != NULL);
|
||||||
|
|
||||||
StorageWrite *this = memNew(sizeof(StorageWrite));
|
StorageWrite *this = NULL;
|
||||||
|
|
||||||
*this = (StorageWrite)
|
OBJ_NEW_BEGIN(StorageWrite, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
.pub =
|
this = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
|
*this = (StorageWrite)
|
||||||
{
|
{
|
||||||
.memContext = memContextCurrent(),
|
.pub =
|
||||||
.interface = interface,
|
{
|
||||||
.io = ioWriteNew(driver, interface->ioInterface),
|
.interface = interface,
|
||||||
},
|
.io = ioWriteNew(driver, interface->ioInterface),
|
||||||
.driver = driver,
|
},
|
||||||
};
|
.driver = objMove(driver, objMemContext(this)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
OBJ_NEW_END();
|
||||||
|
|
||||||
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
FUNCTION_LOG_RETURN(STORAGE_WRITE, this);
|
||||||
}
|
}
|
||||||
|
@ -23,7 +23,7 @@ Functions
|
|||||||
FN_INLINE_ALWAYS StorageWrite *
|
FN_INLINE_ALWAYS StorageWrite *
|
||||||
storageWriteMove(StorageWrite *const this, MemContext *const parentNew)
|
storageWriteMove(StorageWrite *const this, MemContext *const parentNew)
|
||||||
{
|
{
|
||||||
return objMoveContext(this, parentNew);
|
return objMove(this, parentNew);
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
@ -31,7 +31,6 @@ Getters/Setters
|
|||||||
***********************************************************************************************************************************/
|
***********************************************************************************************************************************/
|
||||||
typedef struct StorageWritePub
|
typedef struct StorageWritePub
|
||||||
{
|
{
|
||||||
MemContext *memContext; // Mem context
|
|
||||||
const StorageWriteInterface *interface; // File data (name, driver type, etc.)
|
const StorageWriteInterface *interface; // File data (name, driver type, etc.)
|
||||||
IoWrite *io; // Write interface
|
IoWrite *io; // Write interface
|
||||||
} StorageWritePub;
|
} StorageWritePub;
|
||||||
@ -113,7 +112,7 @@ Destructor
|
|||||||
FN_INLINE_ALWAYS void
|
FN_INLINE_ALWAYS void
|
||||||
storageWriteFree(StorageWrite *const this)
|
storageWriteFree(StorageWrite *const this)
|
||||||
{
|
{
|
||||||
objFreeContext(this);
|
objFree(this);
|
||||||
}
|
}
|
||||||
|
|
||||||
/***********************************************************************************************************************************
|
/***********************************************************************************************************************************
|
||||||
|
@ -145,15 +145,23 @@ testRun(void)
|
|||||||
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleRemote);
|
HRN_CFG_LOAD(cfgCmdArchivePush, argList, .role = cfgCmdRoleRemote);
|
||||||
|
|
||||||
// Create a driver to test remote performance of storageNewItrP() and inject it into storageRepo()
|
// Create a driver to test remote performance of storageNewItrP() and inject it into storageRepo()
|
||||||
StorageTestPerfList driver =
|
StorageTestPerfList *driver = NULL;
|
||||||
|
|
||||||
|
OBJ_NEW_BEGIN(StorageTestPerfList, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
{
|
{
|
||||||
.interface = storageInterfaceTestDummy,
|
driver = OBJ_NEW_ALLOC();
|
||||||
.fileTotal = fileTotal,
|
|
||||||
};
|
|
||||||
|
|
||||||
driver.interface.list = storageTestPerfList;
|
*driver = (StorageTestPerfList)
|
||||||
|
{
|
||||||
|
.interface = storageInterfaceTestDummy,
|
||||||
|
.fileTotal = fileTotal,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
OBJ_NEW_END();
|
||||||
|
|
||||||
Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, &driver, driver.interface);
|
driver->interface.list = storageTestPerfList;
|
||||||
|
|
||||||
|
Storage *storageTest = storageNew(strIdFromZ("test"), STRDEF("/"), 0, 0, false, NULL, driver, driver->interface);
|
||||||
storageHelper.storageRepoWrite = memNew(sizeof(Storage *));
|
storageHelper.storageRepoWrite = memNew(sizeof(Storage *));
|
||||||
storageHelper.storageRepoWrite[0] = storageTest;
|
storageHelper.storageRepoWrite[0] = storageTest;
|
||||||
|
|
||||||
|
@ -249,17 +249,25 @@ testRun(void)
|
|||||||
ASSERT(TEST_SCALE <= 1000000);
|
ASSERT(TEST_SCALE <= 1000000);
|
||||||
|
|
||||||
// Create a storage driver to test manifest build with an arbitrary number of files
|
// Create a storage driver to test manifest build with an arbitrary number of files
|
||||||
StorageTestManifestNewBuild driver =
|
StorageTestManifestNewBuild *driver = NULL;
|
||||||
{
|
|
||||||
.interface = storageInterfaceTestDummy,
|
|
||||||
.fileTotal = 100000 * (unsigned int)TEST_SCALE,
|
|
||||||
};
|
|
||||||
|
|
||||||
driver.interface.info = storageTestManifestNewBuildInfo;
|
OBJ_NEW_BEGIN(StorageTestManifestNewBuild, .childQty = MEM_CONTEXT_QTY_MAX)
|
||||||
driver.interface.list = storageTestManifestNewBuildList;
|
{
|
||||||
|
driver = OBJ_NEW_ALLOC();
|
||||||
|
|
||||||
|
*driver = (StorageTestManifestNewBuild)
|
||||||
|
{
|
||||||
|
.interface = storageInterfaceTestDummy,
|
||||||
|
.fileTotal = 100000 * (unsigned int)TEST_SCALE,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
OBJ_NEW_END();
|
||||||
|
|
||||||
|
driver->interface.info = storageTestManifestNewBuildInfo;
|
||||||
|
driver->interface.list = storageTestManifestNewBuildList;
|
||||||
|
|
||||||
const Storage *const storagePg = storageNew(
|
const Storage *const storagePg = storageNew(
|
||||||
strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
|
strIdFromZ("test"), STRDEF("/pg"), 0, 0, false, NULL, driver, driver->interface);
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("build manifest");
|
TEST_TITLE("build manifest");
|
||||||
@ -280,7 +288,7 @@ testRun(void)
|
|||||||
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
||||||
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
|
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
|
||||||
|
|
||||||
TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, " check file total");
|
TEST_RESULT_UINT(manifestFileTotal(manifest), driver->fileTotal, " check file total");
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("save manifest");
|
TEST_TITLE("save manifest");
|
||||||
@ -310,7 +318,7 @@ testRun(void)
|
|||||||
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
||||||
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
|
// TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
|
||||||
|
|
||||||
TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, " check file total");
|
TEST_RESULT_UINT(manifestFileTotal(manifest), driver->fileTotal, " check file total");
|
||||||
|
|
||||||
// -------------------------------------------------------------------------------------------------------------------------
|
// -------------------------------------------------------------------------------------------------------------------------
|
||||||
TEST_TITLE("find all files");
|
TEST_TITLE("find all files");
|
||||||
|
Reference in New Issue
Block a user