db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-07-03 00:26:59 +02:00
Code Issues Releases Activity

Storage object improvements.

Browse Source 
* Add storageCopy(), storageMove(), and storagePathSync().
* Separate StorageFile object into separate read and write objects.
* Abstract out Posix file read/write objects.
This commit is contained in:
David Steele
2018-04-23 17:26:27 -04:00
parent 02cc8ccbd4
commit bb8c315cad
36 changed files with 1802 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
build/error.yaml
View File

@ -80,6 +80,9 @@ memory: 94
cipher: 95
param-invalid: 96
# Unable to close a path
path-close: 97
# This error should not be thrown directly -- it serves as a parent for the C errors
runtime: 122

2
doc/xml/release.xml
View File

@ -62,7 +62,7 @@
</release-item>
<release-item>
<p>Storage object improvements. Convert all functions to variadic functions. Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function. Add <code>storageExists()</code>, <code>storagePathCreate()</code>, and <code>storageRemove()</code>. Add <code>StorageFile</code> object and <code>storageOpenRead()</code>/<code>storageOpenWrite()</code>. Abstract Posix driver code into a separate module. Add <code>storagePathRemove()</code> and use it in the Perl Posix driver.</p>
<p>Storage object improvements. Convert all functions to variadic functions. Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function. Add <code>storageCopy()</code>, <code>storageExists()</code>, <code>storageMove()</code>, <code>storageNewRead()</code>/<code>storageNewWrite()</code>, <code>storagePathCreate()</code>, <code>storagePathRemove()</code>, <code>storagePathSync()</code>, and <code>storageRemove()</code>. Add <code>StorageFileRead</code> and <code>StorageFileWrite</code> objects. Abstract Posix driver code into a separate module. Call <code>storagePathCreate()</code> from the Perl Posix driver.</p>
</release-item>
<release-item>

2
lib/pgBackRest/Common/ExceptionAuto.pm
View File

@ -161,6 +161,8 @@ use constant ERROR_CIPHER => 95;
push @EXPORT, qw(ERROR_CIPHER);
use constant ERROR_PARAM_INVALID => 96;
push @EXPORT, qw(ERROR_PARAM_INVALID);
use constant ERROR_PATH_CLOSE => 97;
push @EXPORT, qw(ERROR_PATH_CLOSE);
use constant ERROR_RUNTIME => 122;
push @EXPORT, qw(ERROR_RUNTIME);
use constant ERROR_INVALID => 123;

2
libc/LibC.xs
View File

@ -50,7 +50,7 @@ These includes are from the src directory. There is no Perl-specific code in th
#include "config/parse.h"
#include "perl/config.h"
#include "postgres/pageChecksum.h"
#include "storage/driver/posix.h"
#include "storage/driver/posix/driver.h"
/***********************************************************************************************************************************
Helper macros

8
libc/Makefile.PL
View File

@ -102,8 +102,12 @@ my @stryCFile =
'config/parse.c',
'perl/config.c',
'postgres/pageChecksum.c',
'storage/driver/posix.c',
'storage/file.c',
'storage/driver/posix/driver.c',
'storage/driver/posix/driverFile.c',
'storage/driver/posix/driverRead.c',
'storage/driver/posix/driverWrite.c',
'storage/fileRead.c',
'storage/fileWrite.c',
'storage/helper.c',
'storage/storage.c',
);

8
src/Makefile
View File

@ -81,8 +81,12 @@ SRCS = \
config/parse.c \
perl/config.c \
perl/exec.c \
storage/driver/posix.c \
storage/file.c \
storage/driver/posix/driver.c \
storage/driver/posix/driverFile.c \
storage/driver/posix/driverRead.c \
storage/driver/posix/driverWrite.c \
storage/fileRead.c \
storage/fileWrite.c \
storage/helper.c \
storage/storage.c \
main.c

2
src/command/archive/push/push.c
View File

@ -49,7 +49,7 @@ walStatus(const String *walSegment, bool confessOnError)
const String *statusFile = strLstGet(fileList, 0);
String *content = strNewBuf(
storageGetNP(storageOpenReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
// Get the code and message if the file has content
int code = 0;

2
src/common/error.auto.c
View File

@ -78,6 +78,7 @@ ERROR_DEFINE( 93, FileExistsError, RuntimeError);
ERROR_DEFINE( 94, MemoryError, RuntimeError);
ERROR_DEFINE( 95, CipherError, RuntimeError);
ERROR_DEFINE( 96, ParamInvalidError, RuntimeError);
ERROR_DEFINE( 97, PathCloseError, RuntimeError);
ERROR_DEFINE(122, RuntimeError, RuntimeError);
ERROR_DEFINE(123, InvalidError, RuntimeError);
ERROR_DEFINE(124, UnhandledError, RuntimeError);
@ -159,6 +160,7 @@ static const ErrorType *errorTypeList[] =
&MemoryError,
&CipherError,
&ParamInvalidError,
&PathCloseError,
&RuntimeError,
&InvalidError,
&UnhandledError,

1
src/common/error.auto.h
View File

@ -80,6 +80,7 @@ ERROR_DECLARE(FileExistsError);
ERROR_DECLARE(MemoryError);
ERROR_DECLARE(CipherError);
ERROR_DECLARE(ParamInvalidError);
ERROR_DECLARE(PathCloseError);
ERROR_DECLARE(RuntimeError);
ERROR_DECLARE(InvalidError);
ERROR_DECLARE(UnhandledError);

2
src/common/ini.c
View File

@ -212,7 +212,7 @@ iniLoad(Ini *this, const String *fileName)
MEM_CONTEXT_TEMP_BEGIN()
{
iniParse(this, strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), this->fileName))));
iniParse(this, strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), this->fileName))));
}
MEM_CONTEXT_TEMP_END();
}

6
src/config/parse.c
View File

@ -164,7 +164,7 @@ cfgFileLoad( // NOTE: Pas
configFileName = optConfigDefault;
// Load the config file
Buffer *buffer = storageGetNP(storageOpenReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
Buffer *buffer = storageGetNP(storageNewReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
// Convert the contents of the file buffer to the config string object
if (buffer != NULL)
@ -172,7 +172,7 @@ cfgFileLoad( // NOTE: Pas
else if (strEq(configFileName, optConfigDefaultCurrent))
{
// If confg is current default and it was not found, attempt to load the config file from the old default location
buffer = storageGetNP(storageOpenReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
buffer = storageGetNP(storageNewReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
if (buffer != NULL)
result = strNewBuf(buffer);
@ -210,7 +210,7 @@ cfgFileLoad( // NOTE: Pas
for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
{
Buffer *fileBuffer = storageGetNP(
storageOpenReadP(
storageNewReadP(
storageLocal(), strNewFmt("%s/%s", strPtr(configIncludePath), strPtr(strLstGet(list, listIdx))),
.ignoreMissing = true));

188
src/storage/driver/posix.c → src/storage/driver/posix/driver.c
View File

@ -1,29 +1,19 @@
/***********************************************************************************************************************************
Storage Posix Driver
Storage Driver Posix
***********************************************************************************************************************************/
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common/memContext.h"
#include "common/regExp.h"
#include "storage/driver/posix.h"
#include "storage/driver/posix/driver.h"
#include "storage/driver/posix/driverFile.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
Storage file data - holds the file handle.
***********************************************************************************************************************************/
typedef struct StorageFileDataPosix
{
MemContext *memContext;
int handle;
} StorageFileDataPosix;
#define STORAGE_DATA(file) \
((StorageFileDataPosix *)storageFileData(file))
/***********************************************************************************************************************************
Does a file/path exist?
***********************************************************************************************************************************/
@ -48,61 +38,6 @@ storageDriverPosixExists(const String *path)
return result;
}
/***********************************************************************************************************************************
Read from storage into a buffer
***********************************************************************************************************************************/
Buffer *
storageDriverPosixGet(const StorageFile *file)
{
Buffer *result = NULL;
MEM_CONTEXT_TEMP_BEGIN()
{
TRY_BEGIN()
{
size_t bufferSize = storageBufferSize(storageFileStorage(file));
result = bufNew(bufferSize);
// Create result buffer with buffer size
ssize_t actualBytes = 0;
size_t totalBytes = 0;
do
{
// Grow the buffer on subsequent reads
if (totalBytes != 0)
bufResize(result, bufSize(result) + bufferSize);
// Read and handle errors
actualBytes = read(
STORAGE_DATA(file)->handle, bufPtr(result) + totalBytes, bufferSize);
// Error occurred during write
if (actualBytes == -1)
THROW_SYS_ERROR(FileReadError, "unable to read '%s'", strPtr(storageFileName(file)));
// Track total bytes read
totalBytes += (size_t)actualBytes;
}
while (actualBytes != 0);
// Resize buffer to total bytes read
bufResize(result, totalBytes);
}
FINALLY()
{
close(STORAGE_DATA(file)->handle);
storageFileFree(file);
}
TRY_END();
bufMove(result, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
Get a list of files from a directory
***********************************************************************************************************************************/
@ -166,59 +101,59 @@ storageDriverPosixList(const String *path, bool errorOnMissing, const String *ex
}
/***********************************************************************************************************************************
Open a file for reading
Move a file
***********************************************************************************************************************************/
void *
storageDriverPosixOpenRead(const String *file, bool ignoreMissing)
bool
storageDriverPosixMove(StorageFileReadPosix *source, StorageFileWritePosix *destination)
{
StorageFileDataPosix *result = NULL;
bool result = true;
// Open the file and handle errors
int fileHandle = open(strPtr(file), O_RDONLY, 0);
MEM_CONTEXT_TEMP_BEGIN()
{
const String *sourceFile = storageFileReadPosixName(source);
const String *destinationFile = storageFileWritePosixName(destination);
const String *destinationPath = storageFileWritePosixPath(destination);
if (fileHandle == -1)
{
// Error unless ignore missing is specified
if (!ignoreMissing || errno != ENOENT)
THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for read", strPtr(file));
}
// Else create the storage file and data
else
{
MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
// Attempt to move the file
if (rename(strPtr(sourceFile), strPtr(destinationFile)) == -1)
{
result = memNew(sizeof(StorageFileDataPosix));
result->memContext = MEM_CONTEXT_NEW();
result->handle = fileHandle;
// Detemine which file/path is missing
if (errno == ENOENT)
{
if (!storageDriverPosixExists(sourceFile))
THROW_SYS_ERROR(FileMissingError, "unable to move missing file '%s'", strPtr(sourceFile));
if (!storageFileWritePosixCreatePath(destination))
{
THROW_SYS_ERROR(
PathMissingError, "unable to move '%s' to missing path '%s'", strPtr(sourceFile), strPtr(destinationPath));
}
storageDriverPosixPathCreate(destinationPath, false, false, storageFileWritePosixModePath(destination));
result = storageDriverPosixMove(source, destination);
}
// Else the destination is on a different device so a copy will be needed
else if (errno == EXDEV)
{
result = false;
}
else
THROW_SYS_ERROR(FileMoveError, "unable to move '%s' to '%s'", strPtr(sourceFile), strPtr(destinationFile));
}
// Sync paths on success
else
{
// Sync source path if the destination path was synced and the paths are not equal
if (storageFileWritePosixSyncPath(destination))
{
String *sourcePath = strPath(sourceFile);
if (!strEq(destinationPath, sourcePath))
storageDriverPosixPathSync(sourcePath, false);
}
}
MEM_CONTEXT_NEW_END();
}
return result;
}
/***********************************************************************************************************************************
Open a file for writing
***********************************************************************************************************************************/
void *
storageDriverPosixOpenWrite(const String *file, mode_t mode)
{
// Open the file and handle errors
int fileHandle = open(strPtr(file), O_CREAT | O_TRUNC | O_WRONLY, mode);
if (fileHandle == -1)
THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for write", strPtr(file));
// Create the storage file and data
StorageFileDataPosix *result = NULL;
MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
{
result = memNew(sizeof(StorageFileDataPosix));
result->memContext = MEM_CONTEXT_NEW();
result->handle = fileHandle;
}
MEM_CONTEXT_NEW_END();
MEM_CONTEXT_TEMP_END();
return result;
}
@ -291,22 +226,23 @@ storageDriverPosixPathRemove(const String *path, bool errorOnMissing, bool recur
}
/***********************************************************************************************************************************
Write a buffer to storage
Sync a path
***********************************************************************************************************************************/
void
storageDriverPosixPut(const StorageFile *file, const Buffer *buffer)
storageDriverPosixPathSync(const String *path, bool ignoreMissing)
{
TRY_BEGIN()
// Open directory and handle errors
int handle = storageFilePosixOpen(path, O_RDONLY, 0, ignoreMissing, &PathOpenError, "sync");
// On success
if (handle != -1)
{
if (write(STORAGE_DATA(file)->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
THROW_SYS_ERROR(FileWriteError, "unable to write '%s'", strPtr(storageFileName(file)));
// Attempt to sync the directory
storageFilePosixSync(handle, path, &PathSyncError, true);
// Close the directory
storageFilePosixClose(handle, path, &PathCloseError);
}
FINALLY()
{
close(STORAGE_DATA(file)->handle);
storageFileFree(file);
}
TRY_END();
}
/***********************************************************************************************************************************

18
src/storage/driver/posix.h → src/storage/driver/posix/driver.h
View File

@ -1,27 +1,25 @@
/***********************************************************************************************************************************
Storage Posix Driver
Storage Driver Posix
***********************************************************************************************************************************/
#ifndef STORAGE_DRIVER_POSIX_H
#define STORAGE_DRIVER_POSIX_H
#ifndef STORAGE_DRIVER_POSIX_DRIVER_H
#define STORAGE_DRIVER_POSIX_DRIVER_H
#include <sys/types.h>
#include "common/type/buffer.h"
#include "common/type/string.h"
#include "storage/file.h"
#include "storage/storage.h"
#include "common/type/stringList.h"
#include "storage/driver/posix/driverRead.h"
#include "storage/driver/posix/driverWrite.h"
/***********************************************************************************************************************************
Function
***********************************************************************************************************************************/
bool storageDriverPosixExists(const String *path);
Buffer *storageDriverPosixGet(const StorageFile *file);
StringList *storageDriverPosixList(const String *path, bool errorOnMissing, const String *expression);
void *storageDriverPosixOpenRead(const String *file, bool ignoreMissing);
void *storageDriverPosixOpenWrite(const String *file, mode_t mode);
bool storageDriverPosixMove(StorageFileReadPosix *source, StorageFileWritePosix *destination);
void storageDriverPosixPathCreate(const String *path, bool errorOnExists, bool noParentCreate, mode_t mode);
void storageDriverPosixPathRemove(const String *path, bool errorOnMissing, bool recurse);
void storageDriverPosixPut(const StorageFile *file, const Buffer *buffer);
void storageDriverPosixPathSync(const String *path, bool ignoreMissing);
void storageDriverPosixRemove(const String *file, bool errorOnMissing);
#endif

63
src/storage/driver/posix/driverFile.c Normal file
View File

@ -0,0 +1,63 @@
/***********************************************************************************************************************************
Storage File Routines For Posix
***********************************************************************************************************************************/
// So fsync() will work on older glib versions
#ifndef _POSIX_C_SOURCE
#define _POSIX_C_SOURCE 200112L
#endif
#include <fcntl.h>
#include <unistd.h>
#include "common/assert.h"
#include "storage/driver/posix/driverFile.h"
/***********************************************************************************************************************************
Open a file
Returns the handle of the open file, or -1 for reads if the file is missing and -1 for writes if the path is mssing.
***********************************************************************************************************************************/
int
storageFilePosixOpen(
const String *name, int flags, mode_t mode, bool ignoreMissing, const ErrorType *errorType, const char *purpose)
{
int result = -1;
result = open(strPtr(name), flags, mode);
if (result == -1)
{
if (errno != ENOENT || !ignoreMissing)
THROWP_SYS_ERROR(errorType, "unable to open '%s' for %s", strPtr(name), purpose);
}
return result;
}
/***********************************************************************************************************************************
Sync a file/directory handle
***********************************************************************************************************************************/
void
storageFilePosixSync(int handle, const String *name, const ErrorType *errorType, bool closeOnError)
{
if (fsync(handle) == -1)
{
int errNo = errno;
// Close if requested but don't report errors -- we want to report the sync error instead
if (closeOnError)
close(handle);
THROWP_SYS_ERROR_CODE(errNo, errorType, "unable to sync '%s'", strPtr(name));
}
}
/***********************************************************************************************************************************
Close a file/directory handle
***********************************************************************************************************************************/
void
storageFilePosixClose(int handle, const String *name, const ErrorType *errorType)
{
if (close(handle) == -1)
THROWP_SYS_ERROR(errorType, "unable to close '%s'", strPtr(name));
}

18
src/storage/driver/posix/driverFile.h Normal file
View File

@ -0,0 +1,18 @@
/***********************************************************************************************************************************
Storage File Routines For Posix
***********************************************************************************************************************************/
#ifndef STORAGE_DRIVER_POSIX_DRIVERFILE_H
#define STORAGE_DRIVER_POSIX_DRIVERFILE_H
#include "common/error.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
int storageFilePosixOpen(
const String *name, int flags, mode_t mode, bool ignoreMissing, const ErrorType *errorType, const char *purpose);
void storageFilePosixSync(int handle, const String *name, const ErrorType *errorType, bool closeOnError);
void storageFilePosixClose(int handle, const String *name, const ErrorType *errorType);
#endif

165
src/storage/driver/posix/driverRead.c Normal file
View File

@ -0,0 +1,165 @@
/***********************************************************************************************************************************
Storage File Read Driver For Posix
***********************************************************************************************************************************/
#include <fcntl.h>
#include <unistd.h>
#include "common/assert.h"
#include "common/memContext.h"
#include "storage/driver/posix/driverFile.h"
#include "storage/driver/posix/driverRead.h"
/***********************************************************************************************************************************
Storage file structure
***********************************************************************************************************************************/
struct StorageFileReadPosix
{
MemContext *memContext;
String *name;
bool ignoreMissing;
size_t bufferSize;
int handle;
bool eof;
};
/***********************************************************************************************************************************
Create a new file
***********************************************************************************************************************************/
StorageFileReadPosix *
storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize)
{
StorageFileReadPosix *this = NULL;
ASSERT_DEBUG(name != NULL);
ASSERT_DEBUG(bufferSize > 0);
// Create the file object
MEM_CONTEXT_NEW_BEGIN("StorageFileReadPosix")
{
this = memNew(sizeof(StorageFileReadPosix));
this->memContext = MEM_CONTEXT_NEW();
this->name = strDup(name);
this->ignoreMissing = ignoreMissing;
this->bufferSize = bufferSize;
this->handle = -1;
}
MEM_CONTEXT_NEW_END();
return this;
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
bool
storageFileReadPosixOpen(StorageFileReadPosix *this)
{
bool result = false;
ASSERT_DEBUG(this != NULL);
ASSERT_DEBUG(this->handle == -1);
// Open the file and handle errors
this->handle = storageFilePosixOpen(this->name, O_RDONLY, 0, this->ignoreMissing, &FileOpenError, "read");
// On success set free callback to ensure file handle is freed
if (this->handle != -1)
{
memContextCallback(this->memContext, (MemContextCallback)storageFileReadPosixFree, this);
result = true;
}
return result;
}
/***********************************************************************************************************************************
Read from a file
***********************************************************************************************************************************/
Buffer *
storageFileReadPosix(StorageFileReadPosix *this)
{
Buffer *result = NULL;
ASSERT_DEBUG(this != NULL);
// Read if EOF has not been reached
if (!this->eof)
{
result = bufNew(this->bufferSize);
// Read and handle errors
ssize_t actualBytes = read(this->handle, bufPtr(result), this->bufferSize);
// Error occurred during write
if (actualBytes == -1)
THROW_SYS_ERROR(FileReadError, "unable to read '%s'", strPtr(this->name));
// If no data was read then free the buffer and mark the file as EOF
if (actualBytes == 0)
{
this->eof = true;
bufFree(result);
result = NULL;
}
else
bufResize(result, (size_t)actualBytes);
}
return result;
}
/***********************************************************************************************************************************
Close the file
***********************************************************************************************************************************/
void
storageFileReadPosixClose(StorageFileReadPosix *this)
{
ASSERT_DEBUG(this != NULL);
// Close if the file has not already been closed
if (this->handle != -1)
{
// Close the file
storageFilePosixClose(this->handle, this->name, &FileCloseError);
this->handle = -1;
}
}
/***********************************************************************************************************************************
Should a missing file be ignored?
***********************************************************************************************************************************/
bool
storageFileReadPosixIgnoreMissing(StorageFileReadPosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->ignoreMissing;
}
/***********************************************************************************************************************************
File name
***********************************************************************************************************************************/
const String *
storageFileReadPosixName(StorageFileReadPosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->name;
}
/***********************************************************************************************************************************
Free the file
***********************************************************************************************************************************/
void
storageFileReadPosixFree(StorageFileReadPosix *this)
{
if (this != NULL)
{
storageFileReadPosixClose(this);
memContextFree(this->memContext);
}
}

38
src/storage/driver/posix/driverRead.h Normal file
View File

@ -0,0 +1,38 @@
/***********************************************************************************************************************************
Storage File Read Driver For Posix
***********************************************************************************************************************************/
#ifndef STORAGE_DRIVER_POSIX_DRIVERREAD_H
#define STORAGE_DRIVER_POSIX_DRIVERREAD_H
/***********************************************************************************************************************************
Read file object
***********************************************************************************************************************************/
typedef struct StorageFileReadPosix StorageFileReadPosix;
#include "common/type/buffer.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
StorageFileReadPosix *storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
bool storageFileReadPosixOpen(StorageFileReadPosix *this);
Buffer *storageFileReadPosix(StorageFileReadPosix *this);
void storageFileReadPosixClose(StorageFileReadPosix *this);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool storageFileReadPosixIgnoreMissing(StorageFileReadPosix *this);
const String *storageFileReadPosixName(StorageFileReadPosix *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void storageFileReadPosixFree(StorageFileReadPosix *this);
#endif

255
src/storage/driver/posix/driverWrite.c Normal file
View File

@ -0,0 +1,255 @@
/***********************************************************************************************************************************
Storage File Write Driver For Posix
***********************************************************************************************************************************/
#include <fcntl.h>
#include <stdio.h>
#include <unistd.h>
#include "common/assert.h"
#include "common/memContext.h"
#include "storage/driver/posix/driverFile.h"
#include "storage/driver/posix/driverWrite.h"
#include "storage/driver/posix/driver.h"
#include "storage/fileWrite.h"
/***********************************************************************************************************************************
Storage file structure
***********************************************************************************************************************************/
struct StorageFileWritePosix
{
MemContext *memContext;
String *path;
String *name;
String *nameTmp;
mode_t modeFile;
mode_t modePath;
bool noCreatePath;
bool noSyncFile;
bool noSyncPath;
bool noAtomic;
int handle;
};
/***********************************************************************************************************************************
File open constants
Since open is called more than once use constants to make sure these parameters are always the same
***********************************************************************************************************************************/
#define FILE_OPEN_FLAGS (O_CREAT | O_TRUNC | O_WRONLY)
#define FILE_OPEN_PURPOSE "write"
#define FILE_OPEN_ERROR &FileOpenError
/***********************************************************************************************************************************
Create a new file
***********************************************************************************************************************************/
StorageFileWritePosix *
storageFileWritePosixNew(
const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic)
{
StorageFileWritePosix *this = NULL;
ASSERT_DEBUG(name != NULL);
// Create the file
MEM_CONTEXT_NEW_BEGIN("StorageFileWritePosix")
{
this = memNew(sizeof(StorageFileWritePosix));
this->memContext = MEM_CONTEXT_NEW();
this->path = strPath(name);
this->name = strDup(name);
this->nameTmp = noAtomic ? this->name : strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strPtr(name));
this->modeFile = modeFile;
this->modePath = modePath;
this->noCreatePath = noCreatePath;
this->noSyncFile = noSyncFile;
this->noSyncPath = noSyncPath;
this->noAtomic = noAtomic;
this->handle = -1;
}
MEM_CONTEXT_NEW_END();
return this;
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
void
storageFileWritePosixOpen(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
ASSERT_DEBUG(this->handle == -1);
// Open the file and handle errors
this->handle = storageFilePosixOpen(
this->nameTmp, FILE_OPEN_FLAGS, this->modeFile, !this->noCreatePath, FILE_OPEN_ERROR, FILE_OPEN_PURPOSE);
// If path is missing
if (this->handle == -1)
{
// Create the path
storageDriverPosixPathCreate(this->path, false, false, this->modePath);
// Try the open again
this->handle = storageFilePosixOpen(
this->nameTmp, FILE_OPEN_FLAGS, this->modeFile, false, FILE_OPEN_ERROR, FILE_OPEN_PURPOSE);
}
// On success set free callback to ensure file handle is freed
else
memContextCallback(this->memContext, (MemContextCallback)storageFileWritePosixFree, this);
}
/***********************************************************************************************************************************
Write to a file
***********************************************************************************************************************************/
void
storageFileWritePosix(StorageFileWritePosix *this, const Buffer *buffer)
{
ASSERT_DEBUG(this != NULL);
ASSERT_DEBUG(buffer != NULL);
ASSERT_DEBUG(this->handle != -1);
// Write the data
if (write(this->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
THROW_SYS_ERROR(FileWriteError, "unable to write '%s'", strPtr(this->name));
}
/***********************************************************************************************************************************
Close the file
***********************************************************************************************************************************/
void
storageFileWritePosixClose(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
// Close if the file has not already been closed
if (this->handle != -1)
{
// Sync the file
if (!this->noSyncFile)
storageFilePosixSync(this->handle, this->name, &FileSyncError, false);
// Close the file
storageFilePosixClose(this->handle, this->name, &FileCloseError);
// Rename from temp file
if (!this->noAtomic)
{
if (rename(strPtr(this->nameTmp), strPtr(this->name)) == -1)
THROW_SYS_ERROR(FileMoveError, "unable to move '%s' to '%s'", strPtr(this->nameTmp), strPtr(this->name));
}
// Sync the path
if (!this->noSyncPath)
storageDriverPosixPathSync(this->path, false);
// This marks the file as closed
this->handle = -1;
}
}
/***********************************************************************************************************************************
Will the file be written atomically?
For the posix driver this means writing to a temp file first and then renaming once it is closed and synced.
***********************************************************************************************************************************/
bool
storageFileWritePosixAtomic(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return !this->noAtomic;
}
/***********************************************************************************************************************************
Will the path be created for the file if it does not exist?
***********************************************************************************************************************************/
bool
storageFileWritePosixCreatePath(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return !this->noCreatePath;
}
/***********************************************************************************************************************************
Mode for the file to be created
***********************************************************************************************************************************/
mode_t
storageFileWritePosixModeFile(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->modeFile;
}
/***********************************************************************************************************************************
Mode for any paths that are created while writing the file
***********************************************************************************************************************************/
mode_t
storageFileWritePosixModePath(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->modePath;
}
/***********************************************************************************************************************************
File name
***********************************************************************************************************************************/
const String *
storageFileWritePosixName(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->name;
}
/***********************************************************************************************************************************
File path
***********************************************************************************************************************************/
const String *
storageFileWritePosixPath(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return this->path;
}
/***********************************************************************************************************************************
Will the file be synced after it is closed?
***********************************************************************************************************************************/
bool
storageFileWritePosixSyncFile(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return !this->noSyncFile;
}
/***********************************************************************************************************************************
Will the directory be synced to disk after the write is completed?
***********************************************************************************************************************************/
bool
storageFileWritePosixSyncPath(StorageFileWritePosix *this)
{
ASSERT_DEBUG(this != NULL);
return !this->noSyncPath;
}
/***********************************************************************************************************************************
Free the file
***********************************************************************************************************************************/
void
storageFileWritePosixFree(StorageFileWritePosix *this)
{
if (this != NULL)
{
storageFileWritePosixClose(this);
memContextFree(this->memContext);
}
}

47
src/storage/driver/posix/driverWrite.h Normal file
View File

@ -0,0 +1,47 @@
/***********************************************************************************************************************************
Storage File Write Driver For Posix
***********************************************************************************************************************************/
#ifndef STORAGE_DRIVER_POSIX_DRIVERWRITE_H
#define STORAGE_DRIVER_POSIX_DRIVERWRITE_H
#include <sys/types.h>
/***********************************************************************************************************************************
Write file object
***********************************************************************************************************************************/
typedef struct StorageFileWritePosix StorageFileWritePosix;
#include "common/type/buffer.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
StorageFileWritePosix *storageFileWritePosixNew(
const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
void storageFileWritePosixOpen(StorageFileWritePosix *this);
void storageFileWritePosix(StorageFileWritePosix *this, const Buffer *buffer);
void storageFileWritePosixClose(StorageFileWritePosix *this);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
bool storageFileWritePosixAtomic(StorageFileWritePosix *this);
bool storageFileWritePosixCreatePath(StorageFileWritePosix *this);
mode_t storageFileWritePosixModeFile(StorageFileWritePosix *this);
mode_t storageFileWritePosixModePath(StorageFileWritePosix *this);
const String *storageFileWritePosixName(StorageFileWritePosix *this);
const String *storageFileWritePosixPath(StorageFileWritePosix *this);
bool storageFileWritePosixSyncFile(StorageFileWritePosix *this);
bool storageFileWritePosixSyncPath(StorageFileWritePosix *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void storageFileWritePosixFree(StorageFileWritePosix *this);
#endif

35
src/storage/file.h
View File

@ -1,35 +0,0 @@
/***********************************************************************************************************************************
Storage File
***********************************************************************************************************************************/
#ifndef STORAGE_FILE_H
#define STORAGE_FILE_H
/***********************************************************************************************************************************
Storage file object
***********************************************************************************************************************************/
typedef struct StorageFile StorageFile;
/***********************************************************************************************************************************
Types of storage files, i.e. read or write. The storage module does not allow files to be opened for both read and write since this
is generally not supported by object stores.
***********************************************************************************************************************************/
typedef enum
{
storageFileTypeRead,
storageFileTypeWrite,
} StorageFileType;
#include "storage/storage.h"
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
StorageFile *storageFileNew(const Storage *storage, String *name, StorageFileType type, void *data);
void *storageFileData(const StorageFile *this);
const String *storageFileName(const StorageFile *this);
const Storage *storageFileStorage(const StorageFile *this);
void storageFileFree(const StorageFile *this);
#endif

126
src/storage/fileRead.c Normal file
View File

@ -0,0 +1,126 @@
/***********************************************************************************************************************************
Storage File Read
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/memContext.h"
#include "storage/fileRead.h"
/***********************************************************************************************************************************
Storage file structure
***********************************************************************************************************************************/
struct StorageFileRead
{
MemContext *memContext;
StorageFileReadPosix *fileDriver;
};
/***********************************************************************************************************************************
Create a new storage file
***********************************************************************************************************************************/
StorageFileRead *
storageFileReadNew(const String *name, bool ignoreMissing, size_t bufferSize)
{
StorageFileRead *this = NULL;
ASSERT_DEBUG(name != NULL);
MEM_CONTEXT_NEW_BEGIN("StorageFileRead")
{
this = memNew(sizeof(StorageFileRead));
this->memContext = memContextCurrent();
// Call driver function
this->fileDriver = storageFileReadPosixNew(name, ignoreMissing, bufferSize);
}
MEM_CONTEXT_NEW_END();
return this;
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
bool
storageFileReadOpen(StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileReadPosixOpen(this->fileDriver);
}
/***********************************************************************************************************************************
Read data from the file
***********************************************************************************************************************************/
Buffer *
storageFileRead(StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileReadPosix(this->fileDriver);
}
/***********************************************************************************************************************************
Move the file object to a new context
***********************************************************************************************************************************/
StorageFileRead *
storageFileReadMove(StorageFileRead *this, MemContext *parentNew)
{
if (this != NULL)
memContextMove(this->memContext, parentNew);
return this;
}
/***********************************************************************************************************************************
Close the file
***********************************************************************************************************************************/
void
storageFileReadClose(StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
storageFileReadPosixClose(this->fileDriver);
}
/***********************************************************************************************************************************
Get file driver
***********************************************************************************************************************************/
StorageFileReadPosix *
storageFileReadFileDriver(const StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
return this->fileDriver;
}
/***********************************************************************************************************************************
Should a missing file be ignored?
***********************************************************************************************************************************/
bool
storageFileReadIgnoreMissing(const StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileReadPosixIgnoreMissing(this->fileDriver);
}
/***********************************************************************************************************************************
Get file name
***********************************************************************************************************************************/
const String *
storageFileReadName(const StorageFileRead *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileReadPosixName(this->fileDriver);
}
/***********************************************************************************************************************************
Free the file
***********************************************************************************************************************************/
void
storageFileReadFree(StorageFileRead *this)
{
if (this != NULL)
memContextFree(this->memContext);
}

42
src/storage/fileRead.h Normal file
View File

@ -0,0 +1,42 @@
/***********************************************************************************************************************************
Storage File Read
***********************************************************************************************************************************/
#ifndef STORAGE_FILEREAD_H
#define STORAGE_FILEREAD_H
/***********************************************************************************************************************************
Storage file read object
***********************************************************************************************************************************/
typedef struct StorageFileRead StorageFileRead;
#include "common/type/buffer.h"
#include "common/type/string.h"
#include "storage/driver/posix/driverRead.h"
/***********************************************************************************************************************************
Constructor
***********************************************************************************************************************************/
StorageFileRead *storageFileReadNew(const String *name, bool ignoreMissing, size_t bufferSize);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
bool storageFileReadOpen(StorageFileRead *this);
Buffer *storageFileRead(StorageFileRead *this);
void storageFileReadClose(StorageFileRead *this);
StorageFileRead *storageFileReadMove(StorageFileRead *this, MemContext *parentNew);
/***********************************************************************************************************************************
Getters
***********************************************************************************************************************************/
StorageFileReadPosix *storageFileReadFileDriver(const StorageFileRead *this);
bool storageFileReadIgnoreMissing(const StorageFileRead *this);
const String *storageFileReadName(const StorageFileRead *this);
/***********************************************************************************************************************************
Destructor
***********************************************************************************************************************************/
void storageFileReadFree(StorageFileRead *this);
#endif

202
src/storage/fileWrite.c Normal file
View File

@ -0,0 +1,202 @@
/***********************************************************************************************************************************
Storage File Write
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/memContext.h"
#include "storage/fileWrite.h"
/***********************************************************************************************************************************
Storage file structure
***********************************************************************************************************************************/
struct StorageFileWrite
{
MemContext *memContext;
StorageFileWritePosix *fileDriver;
};
/***********************************************************************************************************************************
Create a new storage file
This object expects its context to be created in advance. This is so the calling function can add whatever data it wants without
required multiple functions and contexts to make it safe.
***********************************************************************************************************************************/
StorageFileWrite *
storageFileWriteNew(
const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic)
{
StorageFileWrite *this = NULL;
ASSERT_DEBUG(name != NULL);
// Create the file. The file driver is not created here because we don't want to open the file for write until we know that
// there is a source file.
MEM_CONTEXT_NEW_BEGIN("StorageFileWrite")
{
this = memNew(sizeof(StorageFileWrite));
this->memContext = MEM_CONTEXT_NEW();
this->fileDriver = storageFileWritePosixNew(name, modeFile, modePath, noCreatePath, noSyncFile, noSyncPath, noAtomic);
}
MEM_CONTEXT_NEW_END();
return this;
}
/***********************************************************************************************************************************
Open the file
***********************************************************************************************************************************/
void
storageFileWriteOpen(StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
// Open the file
storageFileWritePosixOpen(this->fileDriver);
}
/***********************************************************************************************************************************
Write to a file
***********************************************************************************************************************************/
void
storageFileWrite(StorageFileWrite *this, const Buffer *buffer)
{
ASSERT_DEBUG(this != NULL);
// Only write if there is data to write
if (buffer != NULL && bufSize(buffer) > 0)
storageFileWritePosix(this->fileDriver, buffer);
}
/***********************************************************************************************************************************
Move the file object to a new context
***********************************************************************************************************************************/
StorageFileWrite *
storageFileWriteMove(StorageFileWrite *this, MemContext *parentNew)
{
if (this != NULL)
memContextMove(this->memContext, parentNew);
return this;
}
/***********************************************************************************************************************************
Close the file
***********************************************************************************************************************************/
void
storageFileWriteClose(StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
storageFileWritePosixClose(this->fileDriver);
}
/***********************************************************************************************************************************
Will the file be written atomically?
Atomic writes means the file will be complete or be missing. Filesystems have different ways to accomplish this.
***********************************************************************************************************************************/
bool
storageFileWriteAtomic(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixAtomic(this->fileDriver);
}
/***********************************************************************************************************************************
Will the path be created if required?
***********************************************************************************************************************************/
bool
storageFileWriteCreatePath(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixCreatePath(this->fileDriver);
}
/***********************************************************************************************************************************
Get file driver
***********************************************************************************************************************************/
StorageFileWritePosix *
storageFileWriteFileDriver(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return this->fileDriver;
}
/***********************************************************************************************************************************
Get file mode
***********************************************************************************************************************************/
mode_t
storageFileWriteModeFile(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixModeFile(this->fileDriver);
}
/***********************************************************************************************************************************
Get path mode
***********************************************************************************************************************************/
mode_t
storageFileWriteModePath(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixModePath(this->fileDriver);
}
/***********************************************************************************************************************************
Get file name
***********************************************************************************************************************************/
const String *
storageFileWriteName(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixName(this->fileDriver);
}
/***********************************************************************************************************************************
Get file path
***********************************************************************************************************************************/
const String *
storageFileWritePath(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixPath(this->fileDriver);
}
/***********************************************************************************************************************************
Will the file be synced after it is closed?
***********************************************************************************************************************************/
bool
storageFileWriteSyncFile(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixSyncFile(this->fileDriver);
}
/***********************************************************************************************************************************
Will the path be synced after the file is closed?
***********************************************************************************************************************************/
bool
storageFileWriteSyncPath(const StorageFileWrite *this)
{
ASSERT_DEBUG(this != NULL);
return storageFileWritePosixSyncPath(this->fileDriver);
}
/***********************************************************************************************************************************
Free the file
***********************************************************************************************************************************/
void
storageFileWriteFree(const StorageFileWrite *this)
{
if (this != NULL)
memContextFree(this->memContext);
}

98
src/storage/file.c → src/storage/fileWrite.h
View File

@ -1,83 +1,57 @@
/***********************************************************************************************************************************
Storage File
Storage File Write
***********************************************************************************************************************************/
#include "common/assert.h"
#include "common/memContext.h"
#include "storage/file.h"
#ifndef STORAGE_FILEWRITE_H
#define STORAGE_FILEWRITE_H
#include <sys/types.h>
/***********************************************************************************************************************************
Storage file structure
Storage file read object
***********************************************************************************************************************************/
struct StorageFile
{
MemContext *memContext;
const Storage *storage;
String *name;
StorageFileType type;
void *data;
};
typedef struct StorageFileWrite StorageFileWrite;
#include "common/type/buffer.h"
#include "common/type/string.h"
#include "storage/driver/posix/driverWrite.h"
#include "version.h"
/***********************************************************************************************************************************
Create a new storage file
This object expects its context to be created in advance. This is so the calling function can add whatever data it wants without
required multiple functions and contexts to make it safe.
Temporary file extension
***********************************************************************************************************************************/
StorageFile *storageFileNew(const Storage *storage, String *name, StorageFileType type, void *data)
{
ASSERT_DEBUG(storage != NULL);
ASSERT_DEBUG(name != NULL);
ASSERT_DEBUG(data != NULL);
StorageFile *this = memNew(sizeof(StorageFile));
this->memContext = memContextCurrent();
this->storage = storage;
this->name = name;
this->type = type;
this->data = data;
return this;
}
#define STORAGE_FILE_TEMP_EXT PGBACKREST_BIN ".tmp"
/***********************************************************************************************************************************
Get file data
Constructor
***********************************************************************************************************************************/
void *
storageFileData(const StorageFile *this)
{
ASSERT_DEBUG(this != NULL);
return this->data;
}
StorageFileWrite *storageFileWriteNew(
const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic);
/***********************************************************************************************************************************
Get file name
Functions
***********************************************************************************************************************************/
const String *
storageFileName(const StorageFile *this)
{
ASSERT_DEBUG(this != NULL);
void storageFileWriteOpen(StorageFileWrite *this);
void storageFileWrite(StorageFileWrite *this, const Buffer *buffer);
void storageFileWriteClose(StorageFileWrite *this);
return this->name;
}
StorageFileWrite *storageFileWriteMove(StorageFileWrite *this, MemContext *parentNew);
/***********************************************************************************************************************************
Get file storage object
Getters
***********************************************************************************************************************************/
const Storage *
storageFileStorage(const StorageFile *this)
{
ASSERT_DEBUG(this != NULL);
return this->storage;
}
bool storageFileWriteAtomic(const StorageFileWrite *this);
bool storageFileWriteCreatePath(const StorageFileWrite *this);
StorageFileWritePosix *storageFileWriteFileDriver(const StorageFileWrite *this);
mode_t storageFileWriteModeFile(const StorageFileWrite *this);
mode_t storageFileWriteModePath(const StorageFileWrite *this);
const String *storageFileWriteName(const StorageFileWrite *this);
const String *storageFileWritePath(const StorageFileWrite *this);
bool storageFileWriteSyncFile(const StorageFileWrite *this);
bool storageFileWriteSyncPath(const StorageFileWrite *this);
/***********************************************************************************************************************************
Free the file
Destructor
***********************************************************************************************************************************/
void
storageFileFree(const StorageFile *this)
{
if (this != NULL)
memContextFree(this->memContext);
}
void storageFileWriteFree(const StorageFileWrite *this);
#endif

164
src/storage/storage.c
View File

@ -6,7 +6,7 @@ Storage Manager
#include "common/assert.h"
#include "common/memContext.h"
#include "common/wait.h"
#include "storage/driver/posix.h"
#include "storage/driver/posix/driver.h"
#include "storage/storage.h"
/***********************************************************************************************************************************
@ -60,12 +60,42 @@ storageNew(const String *path, StorageNewParam param)
}
/***********************************************************************************************************************************
Get storage buffer size
Copy a file
***********************************************************************************************************************************/
size_t
storageBufferSize(const Storage *this)
bool
storageCopy(StorageFileRead *source, StorageFileWrite *destination)
{
return this->bufferSize;
bool result = false;
MEM_CONTEXT_TEMP_BEGIN()
{
// Open source file
if (storageFileReadOpen(source))
{
// Open the destination file now that we know the source file exists and is readable
storageFileWriteOpen(destination);
// Copy data from source to destination
Buffer *read = NULL;
do
{
read = storageFileRead(source);
storageFileWrite(destination, read);
}
while (read != NULL);
// Close the source and destination files
storageFileReadClose(source);
storageFileWriteClose(destination);
// Set result to indicate that the file was copied
result = true;
}
}
MEM_CONTEXT_TEMP_END();
return result;
}
/***********************************************************************************************************************************
@ -104,13 +134,37 @@ storageExists(const Storage *this, const String *pathExp, StorageExistsParam par
Read from storage into a buffer
***********************************************************************************************************************************/
Buffer *
storageGet(const StorageFile *file)
storageGet(StorageFileRead *file)
{
Buffer *result = NULL;
// Call driver function if a file was passed
if (file != NULL)
result = storageDriverPosixGet(file);
ASSERT_DEBUG(file != NULL);
// If the file exists
if (storageFileReadOpen(file))
{
MEM_CONTEXT_TEMP_BEGIN()
{
result = bufNew(0);
Buffer *read = NULL;
do
{
// Read data
read = storageFileRead(file);
// Add to result and free read buffer
bufCat(result, read);
bufFree(read);
}
while (read != NULL);
bufMove(result, MEM_CONTEXT_OLD());
}
MEM_CONTEXT_TEMP_END();
storageFileReadClose(file);
}
return result;
}
@ -136,25 +190,48 @@ storageList(const Storage *this, const String *pathExp, StorageListParam param)
return result;
}
/***********************************************************************************************************************************
Move a file
***********************************************************************************************************************************/
void
storageMove(StorageFileRead *source, StorageFileWrite *destination)
{
ASSERT_DEBUG(!storageFileReadIgnoreMissing(source));
MEM_CONTEXT_TEMP_BEGIN()
{
// If the file can't be moved it will need to be copied
if (!storageDriverPosixMove(storageFileReadFileDriver(source), storageFileWriteFileDriver(destination)))
{
// Perform the copy
storageCopyNP(source, destination);
// Remove the source file
storageDriverPosixRemove(storageFileReadName(source), false);
// Sync source path if the destination path was synced. We know the source and destination paths are different because
// the move did not succeed. This will need updating when drivers other than Posix/CIFS are implemented.
if (storageFileWriteSyncPath(destination))
storageDriverPosixPathSync(strPath(storageFileReadName(source)), false);
}
}
MEM_CONTEXT_TEMP_END();
}
/***********************************************************************************************************************************
Open a file for reading
***********************************************************************************************************************************/
StorageFile *
storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam param)
StorageFileRead *
storageNewRead(const Storage *this, const String *fileExp, StorageNewReadParam param)
{
StorageFile *result = NULL;
StorageFileRead *result = NULL;
MEM_CONTEXT_NEW_BEGIN("StorageFile")
MEM_CONTEXT_NEW_BEGIN("StorageFileRead")
{
String *fileName = storagePathNP(this, fileExp);
// Call driver function
void *data = storageDriverPosixOpenRead(fileName, param.ignoreMissing);
// Free mem contexts if missing files are ignored
if (data != NULL)
result = storageFileNew(this, fileName, storageFileTypeRead, data);
// Create the file
result = storageFileReadMove(storageFileReadNew(fileName, param.ignoreMissing, this->bufferSize), MEM_CONTEXT_OLD());
}
MEM_CONTEXT_NEW_END();
@ -164,23 +241,26 @@ storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam
/***********************************************************************************************************************************
Open a file for writing
***********************************************************************************************************************************/
StorageFile *
storageOpenWrite(const Storage *this, const String *fileExp, StorageOpenWriteParam param)
StorageFileWrite *
storageNewWrite(const Storage *this, const String *fileExp, StorageNewWriteParam param)
{
StorageFileWrite *result = NULL;
ASSERT_STORAGE_ALLOWS_WRITE();
StorageFile *result = NULL;
MEM_CONTEXT_NEW_BEGIN("StorageFile")
MEM_CONTEXT_TEMP_BEGIN()
{
String *fileName = storagePathNP(this, fileExp);
// Call driver function
result = storageFileNew(
this, fileName, storageFileTypeWrite,
storageDriverPosixOpenWrite(fileName, param.mode != 0 ? param.mode : this->modeFile));
// Create the file
result = storageFileWriteMove(
storageFileWriteNew(
fileName, param.modeFile != 0 ? param.modeFile : this->modeFile,
param.modePath != 0 ? param.modePath : this->modePath, param.noCreatePath, param.noSyncFile, param.noSyncPath,
param.noAtomic),
MEM_CONTEXT_OLD());
}
MEM_CONTEXT_NEW_END();
MEM_CONTEXT_TEMP_END();
return result;
}
@ -323,15 +403,33 @@ storagePathRemove(const Storage *this, const String *pathExp, StoragePathRemoveP
MEM_CONTEXT_TEMP_END();
}
/***********************************************************************************************************************************
Sync a path
***********************************************************************************************************************************/
void storagePathSync(const Storage *this, const String *pathExp, StoragePathSyncParam param)
{
ASSERT_STORAGE_ALLOWS_WRITE();
MEM_CONTEXT_TEMP_BEGIN()
{
// Build the path
String *path = storagePathNP(this, pathExp);
// Call driver function
storageDriverPosixPathSync(path, param.ignoreMissing);
}
MEM_CONTEXT_TEMP_END();
}
/***********************************************************************************************************************************
Write a buffer to storage
***********************************************************************************************************************************/
void
storagePut(const StorageFile *file, const Buffer *buffer)
storagePut(StorageFileWrite *file, const Buffer *buffer)
{
// Write data if buffer is not null. Otherwise, an empty file is expected.
if (buffer != NULL)
storageDriverPosixPut(file, buffer);
storageFileWriteOpen(file);
storageFileWrite(file, buffer);
storageFileWriteClose(file);
}
/***********************************************************************************************************************************

89
src/storage/storage.h
View File

@ -13,7 +13,8 @@ typedef struct Storage Storage;
#include "common/type/buffer.h"
#include "common/type/stringList.h"
#include "storage/file.h"
#include "storage/fileRead.h"
#include "storage/fileWrite.h"
/***********************************************************************************************************************************
Default buffer size
@ -29,7 +30,6 @@ Default file and path modes
#define STORAGE_FILE_MODE_DEFAULT 0640
#define STORAGE_PATH_MODE_DEFAULT 0750
/***********************************************************************************************************************************
Path expression callback function type - used to modify paths based on expressions enclosed in <>
***********************************************************************************************************************************/
@ -54,6 +54,14 @@ typedef struct StorageNewParam
Storage *storageNew(const String *path, StorageNewParam param);
/***********************************************************************************************************************************
storageCopy
***********************************************************************************************************************************/
#define storageCopyNP(source, destination) \
storageCopy(source, destination)
bool storageCopy(StorageFileRead *source, StorageFileWrite *destination);
/***********************************************************************************************************************************
storageExists
***********************************************************************************************************************************/
@ -75,7 +83,7 @@ storageGet
#define storageGetNP(file) \
storageGet(file)
Buffer *storageGet(const StorageFile *file);
Buffer *storageGet(StorageFileRead *file);
/***********************************************************************************************************************************
storageList
@ -94,34 +102,47 @@ typedef struct StorageListParam
StringList *storageList(const Storage *this, const String *pathExp, StorageListParam param);
/***********************************************************************************************************************************
storageOpenRead
storageMove
***********************************************************************************************************************************/
typedef struct StorageOpenReadParam
{
bool ignoreMissing;
} StorageOpenReadParam;
#define storageMoveNP(source, destination) \
storageMove(source, destination)
#define storageOpenReadP(this, pathExp, ...) \
storageOpenRead(this, pathExp, (StorageOpenReadParam){__VA_ARGS__})
#define storageOpenReadNP(this, pathExp) \
storageOpenRead(this, pathExp, (StorageOpenReadParam){0})
StorageFile *storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam param);
void storageMove(StorageFileRead *source, StorageFileWrite *destination);
/***********************************************************************************************************************************
storageOpenWrite
storageNewRead
***********************************************************************************************************************************/
typedef struct StorageOpenWriteParam
typedef struct StorageNewReadParam
{
mode_t mode;
} StorageOpenWriteParam;
bool ignoreMissing;
} StorageNewReadParam;
#define storageOpenWriteP(this, pathExp, ...) \
storageOpenWrite(this, pathExp, (StorageOpenWriteParam){__VA_ARGS__})
#define storageOpenWriteNP(this, pathExp) \
storageOpenWrite(this, pathExp, (StorageOpenWriteParam){0})
#define storageNewReadP(this, pathExp, ...) \
storageNewRead(this, pathExp, (StorageNewReadParam){__VA_ARGS__})
#define storageNewReadNP(this, pathExp) \
storageNewRead(this, pathExp, (StorageNewReadParam){0})
StorageFile *storageOpenWrite(const Storage *this, const String *fileExp, StorageOpenWriteParam param);
StorageFileRead *storageNewRead(const Storage *this, const String *fileExp, StorageNewReadParam param);
/***********************************************************************************************************************************
storageNewWrite
***********************************************************************************************************************************/
typedef struct StorageNewWriteParam
{
mode_t modeFile;
mode_t modePath;
bool noCreatePath;
bool noSyncFile;
bool noSyncPath;
bool noAtomic;
} StorageNewWriteParam;
#define storageNewWriteP(this, pathExp, ...) \
storageNewWrite(this, pathExp, (StorageNewWriteParam){__VA_ARGS__})
#define storageNewWriteNP(this, pathExp) \
storageNewWrite(this, pathExp, (StorageNewWriteParam){0})
StorageFileWrite *storageNewWrite(const Storage *this, const String *fileExp, StorageNewWriteParam param);
/***********************************************************************************************************************************
storagePath
@ -164,13 +185,28 @@ typedef struct StoragePathRemoveParam
void storagePathRemove(const Storage *this, const String *pathExp, StoragePathRemoveParam param);
/***********************************************************************************************************************************
storagePathSync
***********************************************************************************************************************************/
typedef struct StoragePathSync
{
bool ignoreMissing;
} StoragePathSyncParam;
#define storagePathSyncP(this, pathExp, ...) \
storagePathSync(this, pathExp, (StoragePathSyncParam){__VA_ARGS__})
#define storagePathSyncNP(this, pathExp) \
storagePathSync(this, pathExp, (StoragePathSyncParam){0})
void storagePathSync(const Storage *this, const String *pathExp, StoragePathSyncParam param);
/***********************************************************************************************************************************
storagePut
***********************************************************************************************************************************/
#define storagePutNP(file, buffer) \
storagePut(file, buffer)
void storagePut(const StorageFile *file, const Buffer *buffer);
void storagePut(StorageFileWrite *file, const Buffer *buffer);
/***********************************************************************************************************************************
storageRemove
@ -195,9 +231,4 @@ storageFree
void storageFree(const Storage *this);
/***********************************************************************************************************************************
Functions
***********************************************************************************************************************************/
size_t storageBufferSize(const Storage *this);
#endif

12
test/define.yaml
View File

@ -466,19 +466,23 @@ module:
# ----------------------------------------------------------------------------------------------------------------------------
- name: file
total: 1
total: 3
c: true
coverage:
storage/file: full
storage/driver/posix/driverFile: full
storage/driver/posix/driverRead: full
storage/driver/posix/driverWrite: full
storage/fileRead: full
storage/fileWrite: full
# ----------------------------------------------------------------------------------------------------------------------------
- name: storage
total: 10
total: 13
c: true
coverage:
storage/driver/posix: full
storage/driver/posix/driver: full
storage/storage: full
# ----------------------------------------------------------------------------------------------------------------------------

8
test/src/common/logTest.c
View File

@ -50,7 +50,7 @@ After the comparison the log is cleared so the next result can be compared.
void
testLogResult(const char *expected)
{
String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stdoutFile))));
String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stdoutFile))));
if (!strEqZ(actual, expected))
THROW(AssertError, "\n\nexpected log:\n\n%s\n\nbut actual log was:\n\n%s\n\n", expected, strPtr(actual));
@ -67,7 +67,7 @@ After the comparison the log is cleared so the next result can be compared.
void
testLogErrResult(const char *expected)
{
String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stderrFile))));
String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stderrFile))));
if (!strEqZ(actual, expected))
THROW(AssertError, "\n\nexpected error log:\n\n%s\n\nbut actual error log was:\n\n%s\n\n", expected, strPtr(actual));
@ -82,12 +82,12 @@ Make sure nothing is left in the log after all tests have completed
void
testLogFinal()
{
String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stdoutFile))));
String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stdoutFile))));
if (!strEqZ(actual, ""))
THROW(AssertError, "\n\nexpected log to be empty but actual log was:\n\n%s\n\n", strPtr(actual));
actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stderrFile))));
actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stderrFile))));
if (!strEqZ(actual, ""))
THROW(AssertError, "\n\nexpected error log to be empty but actual error log was:\n\n%s\n\n", strPtr(actual));

18
test/src/module/archive/pushTest.c
View File

@ -40,28 +40,28 @@ testRun()
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR)));
TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines");
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\n")));
TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0");
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew(BOGUS_STR "\nmessage")));
TEST_ERROR(walStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int");
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("0\nwarning")));
TEST_RESULT_BOOL(walStatus(segment, false), true, "ok file with warning");
testLogResult("P00 WARN: warning");
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
bufNewStr(strNew("25\nerror")));
TEST_RESULT_BOOL(walStatus(segment, false), true, "error status renamed to ok");
testLogResult(
@ -69,7 +69,7 @@ testRun()
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("")));
TEST_ERROR(
walStatus(segment, false), AssertError,
@ -81,7 +81,7 @@ testRun()
TEST_ERROR(walStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content");
storagePutNP(
storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
bufNewStr(strNew("25\nmessage")));
TEST_ERROR(walStatus(segment, true), AssertError, "message");
@ -138,7 +138,7 @@ testRun()
mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750);
mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750);
storagePutNP(storageOpenWriteNP(storageSpool(), errorFile), bufNewStr(strNew("25\n" BOGUS_STR)));
storagePutNP(storageNewWriteNP(storageSpool(), errorFile), bufNewStr(strNew("25\n" BOGUS_STR)));
TEST_ERROR(cmdArchivePush(), AssertError, BOGUS_STR);
@ -147,7 +147,7 @@ testRun()
// Write out a valid ok file and test for success
// -------------------------------------------------------------------------------------------------------------------------
storagePutNP(
storageOpenWriteNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")),
storageNewWriteNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")),
bufNewStr(strNew("")));
TEST_RESULT_VOID(cmdArchivePush(), "successful push");

2
test/src/module/common/iniTest.c
View File

@ -88,7 +88,7 @@ testRun()
"\n"
);
TEST_RESULT_VOID(storagePutNP(storageOpenWriteNP(storageLocalWrite(), fileName), bufNewStr(content)), "put ini to file");
TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), bufNewStr(content)), "put ini to file");
TEST_RESULT_VOID(iniLoad(ini, fileName), "load ini from file");
TEST_RESULT_STR(strPtr(varStr(iniGet(ini, strNew("global"), strNew("compress")))), "y", "get compress");

6
test/src/module/common/logTest.c
View File

@ -128,7 +128,7 @@ testRun()
Storage *storage = storageNewNP(strNew(testPath()));
TEST_RESULT_STR(
strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stdoutFile)))),
strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stdoutFile)))),
"P00 WARN: format 99\n"
"P00 ERROR: [026]: message\n"
"P00 ERROR: [026]: message1\n"
@ -137,7 +137,7 @@ testRun()
// Check stderr
TEST_RESULT_STR(
strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stderrFile)))),
strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stderrFile)))),
"DEBUG: test.c:test_func(): message\n"
"INFO: info message\n"
"WARN: unable to open log file '/BOGUS': Permission denied\n"
@ -146,7 +146,7 @@ testRun()
// Check file
TEST_RESULT_STR(
strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, fileFile)))),
strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, fileFile)))),
"-------------------PROCESS START-------------------\n"
"P00 DEBUG: test.c:test_func(): message\n"
"\n"

26
test/src/module/config/parseTest.c
View File

@ -55,14 +55,14 @@ testRun()
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePut(
storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(
storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(
strNew(
"[global]\n"
"compress-level=3\n"
"spool-path=/path/to/spool\n")));
storagePut(
storageOpenWriteNP(
storageNewWriteNP(
storageLocalWrite(), strNewFmt("%s/global-backup.conf", strPtr(configIncludePath))), bufNewStr(
strNew(
"[global:backup]\n"
@ -77,14 +77,14 @@ testRun()
"buffer-size=65536\n")));
storagePut(
storageOpenWriteNP(storageLocalWrite(), strNewFmt("%s/db-backup.conf", strPtr(configIncludePath))), bufNewStr(
storageNewWriteNP(storageLocalWrite(), strNewFmt("%s/db-backup.conf", strPtr(configIncludePath))), bufNewStr(
strNew(
"[db:backup]\n"
"compress=n\n"
"recovery-option=a=b\n")));
storagePut(
storageOpenWriteNP(storageLocalWrite(), strNewFmt("%s/stanza.db.conf", strPtr(configIncludePath))), bufNewStr(
storageNewWriteNP(storageLocalWrite(), strNewFmt("%s/stanza.db.conf", strPtr(configIncludePath))), bufNewStr(
strNew(
"[db]\n"
"pg1-host=db\n"
@ -215,7 +215,7 @@ testRun()
mkdir(strPtr(strPath(oldConfigDefault)), 0750);
storagePut(
storageOpenWriteNP(
storageNewWriteNP(
storageLocalWrite(), oldConfigDefault), bufNewStr(
strNew(
"[global:backup]\n"
@ -747,7 +747,7 @@ testRun()
strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global]\n"
"compress=bogus\n"
)));
@ -762,7 +762,7 @@ testRun()
strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global]\n"
"compress=\n"
)));
@ -778,7 +778,7 @@ testRun()
strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[db]\n"
"pg1-path=/path/to/db\n"
"db-path=/also/path/to/db\n"
@ -795,7 +795,7 @@ testRun()
strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[db]\n"
"pg1-path=/path/to/db\n"
"pg1-path=/also/path/to/db\n"
@ -897,7 +897,7 @@ testRun()
strLstAdd(argList, strNew("--reset-backup-standby"));
strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global]\n"
"compress-level=3\n"
"spool-path=/path/to/spool\n"
@ -958,7 +958,7 @@ testRun()
strLstAdd(argList, strNew("--archive-push-queue-max=4503599627370496"));
strLstAdd(argList, strNew("archive-push"));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global]\n"
"spool-path=/path/to/spool\n"
)));
@ -1029,7 +1029,7 @@ testRun()
strLstAdd(argList, strNew("--stanza=db"));
strLstAdd(argList, strNew(TEST_COMMAND_RESTORE));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global:restore]\n"
"recovery-option=f=g\n"
"recovery-option=hijk=l\n"
@ -1051,7 +1051,7 @@ testRun()
strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
strLstAdd(argList, strNew("info"));
storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
"[global]\n"
"repo1-path=/path/to/repo\n"
"\n"

2
test/src/module/help/helpTest.c
View File

@ -305,6 +305,6 @@ testRun()
dup2(stdoutSave, STDOUT_FILENO);
Storage *storage = storageNewNP(strNew(testPath()));
TEST_RESULT_STR(strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stdoutFile)))), generalHelp, " check text");
TEST_RESULT_STR(strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stdoutFile)))), generalHelp, " check text");
}
}

284
test/src/module/storage/fileTest.c
View File

@ -1,6 +1,22 @@
/***********************************************************************************************************************************
Test Storage File
***********************************************************************************************************************************/
#include "storage/storage.h"
/***********************************************************************************************************************************
Get the mode of a file on local storage
***********************************************************************************************************************************/
mode_t
storageStatMode(const String *path)
{
// Attempt to stat the file
struct stat statFile;
if (stat(strPtr(path), &statFile) == -1) // {uncovered - error should not happen}
THROW_SYS_ERROR(FileOpenError, "unable to stat '%s'", strPtr(path)); // {uncovered+}
return statFile.st_mode & 0777;
}
/***********************************************************************************************************************************
Test Run
@ -8,26 +24,266 @@ Test Run
void
testRun()
{
// Create default storage object for testing
Storage *storageTest = storageNewP(strNew(testPath()), .write = true, .bufferSize = 2);
// Create a directory and file that cannot be accessed to test permissions errors
String *fileNoPerm = strNewFmt("%s/noperm/noperm", testPath());
String *pathNoPerm = strPath(fileNoPerm);
TEST_RESULT_INT(
system(
strPtr(strNewFmt("sudo mkdir -m 700 %s && sudo touch %s && sudo chmod 600 %s", strPtr(pathNoPerm), strPtr(fileNoPerm),
strPtr(fileNoPerm)))),
0, "create no perm path/file");
// *****************************************************************************************************************************
if (testBegin("storageFileNew() and storageFileFree()"))
if (testBegin("StorageFile"))
{
MemContext *memContext = NULL;
StorageFile *file = NULL;
TEST_ERROR_FMT(
storageFilePosixOpen(pathNoPerm, O_RDONLY, 0, false, &PathOpenError, "test"), PathOpenError,
"unable to open '%s' for test: [13] Permission denied", strPtr(pathNoPerm));
MEM_CONTEXT_NEW_BEGIN("TestFile")
// -------------------------------------------------------------------------------------------------------------------------
String *fileName = strNewFmt("%s/test.file", testPath());
TEST_ERROR_FMT(
storageFilePosixOpen(fileName, O_RDONLY, 0, false, &FileOpenError, "read"), FileOpenError,
"unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
TEST_RESULT_INT(storageFilePosixOpen(fileName, O_RDONLY, 0, true, &FileOpenError, "read"), -1, "missing file ignored");
// -------------------------------------------------------------------------------------------------------------------------
int handle = -1;
TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileName)))), 0, "create read file");
TEST_ASSIGN(handle, storageFilePosixOpen(fileName, O_RDONLY, 0, false, &FileOpenError, "read"), "open read file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT(
storageFilePosixSync(-99, fileName, &PathSyncError, false), PathSyncError,
"unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_ERROR_FMT(
storageFilePosixSync(-99, fileName, &FileSyncError, true), FileSyncError,
"unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_RESULT_VOID(storageFilePosixSync(handle, fileName, &FileSyncError, false), "sync file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR_FMT(
storageFilePosixClose(-99, fileName, &FileCloseError), FileCloseError,
"unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_RESULT_VOID(storageFilePosixClose(handle, fileName, &FileSyncError), "close file");
TEST_RESULT_INT(system(strPtr(strNewFmt("rm %s", strPtr(fileName)))), 0, "remove read file");
}
// *****************************************************************************************************************************
if (testBegin("StorageFileRead"))
{
StorageFileRead *file = NULL;
TEST_ASSIGN(file, storageNewReadP(storageTest, fileNoPerm, .ignoreMissing = true), "new read file");
TEST_RESULT_PTR(storageFileReadFileDriver(file), file->fileDriver, " check file driver");
TEST_RESULT_BOOL(storageFileReadIgnoreMissing(file), true, " check ignore missing");
TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileNoPerm), " check name");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageNewReadNP(storageTest, fileNoPerm), "new no perm read file");
TEST_ERROR_FMT(
storageFileReadOpen(file), FileOpenError,
"unable to open '%s' for read: [13] Permission denied", strPtr(fileNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
String *fileName = strNewFmt("%s/test.file", testPath());
TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new missing read file");
TEST_ERROR_FMT(
storageFileReadOpen(file), FileOpenError,
"unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageNewReadP(storageTest, fileName, .ignoreMissing = true), "new missing read file");
TEST_RESULT_BOOL(storageFileReadOpen(file), false, " missing file ignored");
// -------------------------------------------------------------------------------------------------------------------------
Buffer *expectedBuffer = bufNewStr(strNew("TESTFILE\n"));
TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, fileName), expectedBuffer), "write test file");
TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new read file");
TEST_RESULT_BOOL(storageFileReadOpen(file), true, " open file");
// Close the file handle so operations will fail
close(file->fileDriver->handle);
TEST_ERROR_FMT(
storageFileRead(file), FileReadError,
"unable to read '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_ERROR_FMT(
storageFileReadClose(file), FileCloseError,
"unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
// Set file handle to -1 so the close on free with not fail
file->fileDriver->handle = -1;
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNew(0);
MEM_CONTEXT_TEMP_BEGIN()
{
memContext = MEM_CONTEXT_NEW();
TEST_ASSIGN(file, storageFileNew((Storage *)1, strNew("file"), storageFileTypeRead, (void *)2), "new file");
TEST_ASSIGN(file, storageFileReadMove(storageNewReadNP(storageTest, fileName), MEM_CONTEXT_OLD()), "new read file");
}
MEM_CONTEXT_NEW_END();
MEM_CONTEXT_TEMP_END();
TEST_RESULT_PTR(storageFileData(file), (void *)2, " check data");
TEST_RESULT_PTR(file->memContext, memContext, " check mem context");
TEST_RESULT_STR(strPtr(storageFileName(file)), "file", " check name");
TEST_RESULT_INT(file->type, storageFileTypeRead, " check type");
TEST_RESULT_PTR(storageFileStorage(file), (Storage *)1, " check storage");
TEST_RESULT_BOOL(storageFileReadOpen(file), true, " open file");
TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileName), " check file name");
TEST_RESULT_VOID(storageFileFree(file), "free file");
TEST_RESULT_VOID(storageFileFree(NULL), "free null file");
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), false, " check file contents (not all loaded yet)");
TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), " load data");
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, " check file contents (all loaded)");
TEST_RESULT_PTR(storageFileRead(file), NULL, " eof");
TEST_RESULT_PTR(storageFileRead(file), NULL, " still eof");
TEST_RESULT_VOID(storageFileReadClose(file), " close file");
TEST_RESULT_VOID(storageFileReadClose(file), " close again");
TEST_RESULT_VOID(storageFileReadFree(file), " free file");
TEST_RESULT_VOID(storageFileReadFree(NULL), " free null file");
TEST_RESULT_VOID(storageFileReadPosixFree(NULL), " free null file");
TEST_RESULT_VOID(storageFileReadMove(NULL, NULL), " move null file");
}
// *****************************************************************************************************************************
if (testBegin("StorageFileWrite"))
{
StorageFileWrite *file = NULL;
TEST_ASSIGN(
file,
storageNewWriteP(
storageTest, fileNoPerm, .modeFile = 0444, .modePath = 0555, .noCreatePath = true, .noSyncFile = true,
.noSyncPath = true, .noAtomic = true),
"new write file");
TEST_RESULT_BOOL(storageFileWriteAtomic(file), false, " check atomic");
TEST_RESULT_BOOL(storageFileWriteCreatePath(file), false, " check create path");
TEST_RESULT_PTR(storageFileWriteFileDriver(file), file->fileDriver, " check file driver");
TEST_RESULT_INT(storageFileWriteModeFile(file), 0444, " check mode file");
TEST_RESULT_INT(storageFileWriteModePath(file), 0555, " check mode path");
TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileNoPerm), " check name");
TEST_RESULT_STR(strPtr(storageFileWritePath(file)), strPtr(strPath(fileNoPerm)), " check path");
TEST_RESULT_BOOL(storageFileWriteSyncPath(file), false, " check sync path");
TEST_RESULT_BOOL(storageFileWriteSyncFile(file), false, " check sync file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileNoPerm, .noAtomic = true), "new write file");
TEST_ERROR_FMT(
storageFileWriteOpen(file), FileOpenError,
"unable to open '%s' for write: [13] Permission denied", strPtr(fileNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
String *fileName = strNewFmt("%s/sub1/test.file", testPath());
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileName, .noCreatePath = true, .noAtomic = true), "new write file");
TEST_ERROR_FMT(
storageFileWriteOpen(file), FileOpenError,
"unable to open '%s' for write: [2] No such file or directory", strPtr(fileName));
// -------------------------------------------------------------------------------------------------------------------------
String *fileTmp = strNewFmt("%s.pgbackrest.tmp", strPtr(fileName));
Buffer *buffer = bufNewStr(strNew("TESTFILE\n"));
TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file");
TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileName), " check file name");
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
// Close the file handle so operations will fail
close(file->fileDriver->handle);
storageRemoveP(storageTest, fileTmp, .errorOnMissing = true);
TEST_ERROR_FMT(
storageFileWrite(file, buffer), FileWriteError,
"unable to write '%s': [9] Bad file descriptor", strPtr(fileName));
TEST_ERROR_FMT(
storageFileWriteClose(file), FileSyncError,
"unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
// Disable file sync so the close can be reached
file->fileDriver->noSyncFile = true;
TEST_ERROR_FMT(
storageFileWriteClose(file), FileCloseError,
"unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
// Set file handle to -1 so the close on free with not fail
file->fileDriver->handle = -1;
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file");
TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileName), " check file name");
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
// Rename the file back to original name from tmp -- this will cause the rename in close to fail
TEST_RESULT_INT(rename(strPtr(fileTmp), strPtr(fileName)), 0, " rename tmp file");
TEST_ERROR_FMT(
storageFileWriteClose(file), FileMoveError,
"unable to move '%s' to '%s': [2] No such file or directory", strPtr(fileTmp), strPtr(fileName));
// Set file handle to -1 so the close on free with not fail
file->fileDriver->handle = -1;
storageRemoveP(storageTest, fileName, .errorOnMissing = true);
// -------------------------------------------------------------------------------------------------------------------------
MEM_CONTEXT_TEMP_BEGIN()
{
TEST_ASSIGN(file, storageFileWriteMove(storageNewWriteNP(storageTest, fileName), MEM_CONTEXT_OLD()), "new write file");
}
MEM_CONTEXT_TEMP_END();
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
TEST_RESULT_VOID(storageFileWrite(file, NULL), " write null buffer to file");
TEST_RESULT_VOID(storageFileWrite(file, bufNew(0)), " write zero buffer to file");
TEST_RESULT_VOID(storageFileWrite(file, buffer), " write to file");
TEST_RESULT_VOID(storageFileWriteClose(file), " close file");
TEST_RESULT_VOID(storageFileWriteFree(file), " free file");
TEST_RESULT_VOID(storageFileWriteFree(NULL), " free null file");
TEST_RESULT_VOID(storageFileWritePosixFree(NULL), " free null posix file");
TEST_RESULT_VOID(storageFileWriteMove(NULL, NULL), " move null file");
Buffer *expectedBuffer = storageGetNP(storageNewReadNP(storageTest, fileName));
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, " check file contents");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0750, " check path mode");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, " check file mode");
storageRemoveP(storageTest, fileName, .errorOnMissing = true);
// -------------------------------------------------------------------------------------------------------------------------
fileName = strNewFmt("%s/sub2/test.file", testPath());
TEST_ASSIGN(
file,
storageNewWriteP(
storageTest, fileName, .modePath = 0700, .modeFile = 0600, .noSyncPath = true, .noSyncFile = true,
.noAtomic = true),
"new write file");
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
TEST_RESULT_VOID(storageFileWrite(file, buffer), " write to file");
TEST_RESULT_VOID(storageFileWriteClose(file), " close file");
expectedBuffer = storageGetNP(storageNewReadNP(storageTest, fileName));
TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, " check file contents");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0700, " check path mode");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0600, " check file mode");
storageRemoveP(storageTest, fileName, .errorOnMissing = true);
}
}

6
test/src/module/storage/helperTest.c
View File

@ -22,7 +22,7 @@ testRun()
TEST_RESULT_STR(strPtr(storagePathNP(storage, NULL)), "/", "check base path");
TEST_ERROR(storageOpenWriteNP(storage, writeFile), AssertError, "assertion 'this->write == true' failed");
TEST_ERROR(storageNewWriteNP(storage, writeFile), AssertError, "assertion 'this->write == true' failed");
}
// -----------------------------------------------------------------------------------------------------------------------------
if (testBegin("storageLocalWrite()"))
@ -36,7 +36,7 @@ testRun()
TEST_RESULT_STR(strPtr(storagePathNP(storage, NULL)), "/", "check base path");
TEST_RESULT_VOID(storageOpenWriteNP(storage, writeFile), "writes are allowed");
TEST_RESULT_VOID(storageNewWriteNP(storage, writeFile), "writes are allowed");
}
// -----------------------------------------------------------------------------------------------------------------------------
@ -65,6 +65,6 @@ testRun()
TEST_ERROR(storagePathNP(storage, strNew("<" BOGUS_STR ">")), AssertError, "invalid expression '<BOGUS>'");
TEST_RESULT_VOID(storageOpenWriteNP(storage, writeFile), "writes are allowed");
TEST_RESULT_VOID(storageNewWriteNP(storage, writeFile), "writes are allowed");
}
}

288
test/src/module/storage/storageTest.c
View File

@ -2,7 +2,8 @@
Test Storage Manager
***********************************************************************************************************************************/
#include "common/time.h"
#include "storage/file.h"
#include "storage/fileRead.h"
#include "storage/fileWrite.h"
/***********************************************************************************************************************************
Get the mode of a file on local storage
@ -42,7 +43,8 @@ void
testRun()
{
// Create default storage object for testing
Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
Storage *storageTest = storageNewP(strNew(testPath()), .write = true, .bufferSize = 3);
Storage *storageTmp = storageNewP(strNew("/tmp"), .write = true);
// Create a directory and file that cannot be accessed to test permissions errors
String *fileNoPerm = strNewFmt("%s/noperm/noperm", testPath());
@ -52,7 +54,7 @@ testRun()
system(
strPtr(strNewFmt("sudo mkdir -m 700 %s && sudo touch %s && sudo chmod 600 %s", strPtr(pathNoPerm), strPtr(fileNoPerm),
strPtr(fileNoPerm)))),
0, "create no perm file");
0, "create no perm path/file");
// *****************************************************************************************************************************
if (testBegin("storageNew() and storageFree()"))
@ -94,9 +96,9 @@ testRun()
TEST_RESULT_BOOL(storageExistsP(storageTest, strNew("missing"), .timeout = .1), false, "file does not exist");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
TEST_ERROR_FMT(
storageExistsNP(storageTest, fileNoPerm), FileOpenError,
strPtr(strNewFmt("unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm))));
"unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
String *fileExists = strNewFmt("%s/exists", testPath());
@ -121,35 +123,149 @@ testRun()
if (testBegin("storageList()"))
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
TEST_ERROR_FMT(
storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathOpenError,
strPtr(strNewFmt("unable to open path '%s/BOGUS' for read: [2] No such file or directory", testPath())));
"unable to open path '%s/BOGUS' for read: [2] No such file or directory", testPath());
TEST_RESULT_PTR(storageListNP(storageTest, strNew(BOGUS_STR)), NULL, "ignore missing dir");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
TEST_ERROR_FMT(
storageListNP(storageTest, pathNoPerm), PathOpenError,
strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm))));
"unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm));
// Should still error even when ignore missing
TEST_ERROR(
TEST_ERROR_FMT(
storageListNP(storageTest, pathNoPerm), PathOpenError,
strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm))));
"unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storagePutNP(storageOpenWriteNP(storageTest, strNew("aaa.txt")), bufNewStr(strNew("aaa"))), "write aaa.text");
storagePutNP(storageNewWriteNP(storageTest, strNew("aaa.txt")), bufNewStr(strNew("aaa"))), "write aaa.text");
TEST_RESULT_STR(
strPtr(strLstJoin(storageListNP(storageTest, NULL), ", ")), "aaa.txt, stderr.log, stdout.log, noperm",
"dir list");
TEST_RESULT_VOID(
storagePutNP(storageOpenWriteNP(storageTest, strNew("bbb.txt")), bufNewStr(strNew("bbb"))), "write bbb.text");
storagePutNP(storageNewWriteNP(storageTest, strNew("bbb.txt")), bufNewStr(strNew("bbb"))), "write bbb.text");
TEST_RESULT_STR(
strPtr(strLstJoin(storageListP(storageTest, NULL, .expression = strNew("^bbb")), ", ")), "bbb.txt", "dir list");
}
// *****************************************************************************************************************************
if (testBegin("storageCopy()"))
{
String *sourceFile = strNewFmt("%s/source.txt", testPath());
String *destinationFile = strNewFmt("%s/destination.txt", testPath());
StorageFileRead *source = storageNewReadNP(storageTest, sourceFile);
StorageFileWrite *destination = storageNewWriteNP(storageTest, destinationFile);
TEST_ERROR_FMT(
storageCopyNP(source, destination), FileOpenError,
"unable to open '%s' for read: [2] No such file or directory", strPtr(sourceFile));
// -------------------------------------------------------------------------------------------------------------------------
source = storageNewReadP(storageTest, sourceFile, .ignoreMissing = true);
TEST_RESULT_BOOL(storageCopyNP(source, destination), false, "copy and ignore missing file");
// -------------------------------------------------------------------------------------------------------------------------
Buffer *expectedBuffer = bufNewStr(strNew("TESTFILE\n"));
TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, sourceFile), expectedBuffer), "write source file");
source = storageNewReadNP(storageTest, sourceFile);
TEST_RESULT_BOOL(storageCopyNP(source, destination), true, "copy file");
TEST_RESULT_BOOL(bufEq(expectedBuffer, storageGetNP(storageNewReadNP(storageTest, destinationFile))), true, "check file");
storageRemoveP(storageTest, sourceFile, .errorOnMissing = true);
storageRemoveP(storageTest, destinationFile, .errorOnMissing = true);
}
// *****************************************************************************************************************************
if (testBegin("storageMove()"))
{
String *sourceFile = strNewFmt("%s/source.txt", testPath());
String *destinationFile = strNewFmt("%s/sub/destination.txt", testPath());
StorageFileRead *source = storageNewReadNP(storageTest, sourceFile);
StorageFileWrite *destination = storageNewWriteNP(storageTest, destinationFile);
TEST_ERROR_FMT(
storageMoveNP(source, destination), FileMissingError,
"unable to move missing file '%s': [2] No such file or directory", strPtr(sourceFile));
// -------------------------------------------------------------------------------------------------------------------------
source = storageNewReadNP(storageTest, fileNoPerm);
TEST_ERROR_FMT(
storageMoveNP(source, destination), FileMoveError,
"unable to move '%s' to '%s': [13] Permission denied", strPtr(fileNoPerm), strPtr(destinationFile));
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNewStr(strNew("TESTFILE"));
storagePutNP(storageNewWriteNP(storageTest, sourceFile), buffer);
source = storageNewReadNP(storageTest, sourceFile);
destination = storageNewWriteP(storageTest, destinationFile, .noCreatePath = true);
TEST_ERROR_FMT(
storageMoveNP(source, destination), PathMissingError,
"unable to move '%s' to missing path '%s': [2] No such file or directory", strPtr(sourceFile),
strPtr(strPath(destinationFile)));
// -------------------------------------------------------------------------------------------------------------------------
destination = storageNewWriteNP(storageTest, destinationFile);
TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to subpath");
TEST_RESULT_BOOL(storageExistsNP(storageTest, sourceFile), false, "check source file not exists");
TEST_RESULT_BOOL(storageExistsNP(storageTest, destinationFile), true, "check destination file exists");
TEST_RESULT_STR(
strPtr(strNewBuf(storageGetNP(storageNewReadNP(storageTest, destinationFile)))), "TESTFILE",
"check destination contents");
// -------------------------------------------------------------------------------------------------------------------------
sourceFile = destinationFile;
source = storageNewReadNP(storageTest, sourceFile);
destinationFile = strNewFmt("%s/sub/destination2.txt", testPath());
destination = storageNewWriteNP(storageTest, destinationFile);
TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to same path");
// -------------------------------------------------------------------------------------------------------------------------
sourceFile = destinationFile;
source = storageNewReadNP(storageTest, sourceFile);
destinationFile = strNewFmt("%s/source.txt", testPath());
destination = storageNewWriteP(storageTest, destinationFile, .noSyncPath = true);
TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to parent path (no sync)");
// Move across filesystems
// -------------------------------------------------------------------------------------------------------------------------
sourceFile = destinationFile;
source = storageNewReadNP(storageTest, sourceFile);
destinationFile = strNewFmt("/tmp/destination.txt");
destination = storageNewWriteNP(storageTmp, destinationFile);
TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to another filesystem");
TEST_RESULT_BOOL(storageExistsNP(storageTest, sourceFile), false, "check source file not exists");
TEST_RESULT_BOOL(storageExistsNP(storageTmp, destinationFile), true, "check destination file exists");
// Move across fileystems without syncing the paths
// -------------------------------------------------------------------------------------------------------------------------
sourceFile = destinationFile;
source = storageNewReadNP(storageTmp, sourceFile);
destinationFile = strNewFmt("%s/source.txt", testPath());
destination = storageNewWriteP(storageTest, destinationFile, .noSyncPath = true);
TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to another filesystem without path sync");
TEST_RESULT_BOOL(storageExistsNP(storageTmp, sourceFile), false, "check source file not exists");
TEST_RESULT_BOOL(storageExistsNP(storageTest, destinationFile), true, "check destination file exists");
storageRemoveP(storageTest, destinationFile, .errorOnMissing = true);
}
// *****************************************************************************************************************************
if (testBegin("storagePath()"))
{
@ -200,16 +316,16 @@ testRun()
TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub1")), "create sub1");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strNew("sub1"))), 0750, "check sub1 dir mode");
TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub1")), "create sub1 again");
TEST_ERROR(
TEST_ERROR_FMT(
storagePathCreateP(storageTest, strNew("sub1"), .errorOnExists = true), PathCreateError,
strPtr(strNewFmt("unable to create path '%s/sub1': [17] File exists", testPath())));
"unable to create path '%s/sub1': [17] File exists", testPath());
TEST_RESULT_VOID(storagePathCreateP(storageTest, strNew("sub2"), .mode = 0777), "create sub2 with custom mode");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strNew("sub2"))), 0777, "check sub2 dir mode");
TEST_ERROR(
TEST_ERROR_FMT(
storagePathCreateP(storageTest, strNew("sub3/sub4"), .noParentCreate = true), PathCreateError,
strPtr(strNewFmt("unable to create path '%s/sub3/sub4': [2] No such file or directory", testPath())));
"unable to create path '%s/sub3/sub4': [2] No such file or directory", testPath());
TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub3/sub4")), "create sub3/sub4");
TEST_RESULT_INT(system(strPtr(strNewFmt("rm -rf %s/sub*", testPath()))), 0, "remove sub paths");
@ -221,9 +337,9 @@ testRun()
// -------------------------------------------------------------------------------------------------------------------------
String *pathRemove1 = strNewFmt("%s/remove1", testPath());
TEST_ERROR(
TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove1, .errorOnMissing = true), PathRemoveError,
strPtr(strNewFmt("unable to remove path '%s': [2] No such file or directory", strPtr(pathRemove1))));
"unable to remove path '%s': [2] No such file or directory", strPtr(pathRemove1));
TEST_RESULT_VOID(storagePathRemoveP(storageTest, pathRemove1, .recurse = true), "ignore missing path");
// -------------------------------------------------------------------------------------------------------------------------
@ -231,19 +347,19 @@ testRun()
TEST_RESULT_INT(system(strPtr(strNewFmt("sudo mkdir -p -m 700 %s", strPtr(pathRemove2)))), 0, "create noperm paths");
TEST_ERROR(
TEST_ERROR_FMT(
storagePathRemoveNP(storageTest, pathRemove2), PathRemoveError,
strPtr(strNewFmt("unable to remove path '%s': [13] Permission denied", strPtr(pathRemove2))));
TEST_ERROR(
"unable to remove path '%s': [13] Permission denied", strPtr(pathRemove2));
TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2))));
"unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2));
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove1)))), 0, "top path can be removed");
TEST_ERROR(
TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2))));
"unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2));
// -------------------------------------------------------------------------------------------------------------------------
String *fileRemove = strNewFmt("%s/remove.txt", strPtr(pathRemove2));
@ -254,9 +370,9 @@ testRun()
strPtr(fileRemove)))),
0, "add no perm file");
TEST_ERROR(
TEST_ERROR_FMT(
storagePathRemoveP(storageTest, pathRemove1, .recurse = true), PathRemoveError,
strPtr(strNewFmt("unable to remove path/file '%s': [13] Permission denied", strPtr(fileRemove))));
"unable to remove path/file '%s': [13] Permission denied", strPtr(fileRemove));
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove2)))), 0, "bottom path can be removed");
@ -276,46 +392,72 @@ testRun()
}
// *****************************************************************************************************************************
if (testBegin("storageOpenRead()"))
if (testBegin("storagePathSync()"))
{
TEST_ERROR(
storageOpenReadNP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR)),
FileOpenError,
strPtr(strNewFmt("unable to open '%s/%s' for read: [2] No such file or directory", testPath(), BOGUS_STR)));
TEST_RESULT_PTR(
storageOpenReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true),
NULL, "open missing file without error");
TEST_ERROR_FMT(
storagePathSyncNP(storageTest, fileNoPerm), PathOpenError,
"unable to open '%s' for sync: [13] Permission denied", strPtr(fileNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
storageGetNP(storageOpenReadP(storageTest, fileNoPerm, .ignoreMissing = true)), FileOpenError,
strPtr(strNewFmt("unable to open '%s' for read: [13] Permission denied", strPtr(fileNoPerm))));
String *pathName = strNewFmt("%s/testpath", testPath());
TEST_ERROR_FMT(
storagePathSyncNP(storageTest, pathName), PathOpenError,
"unable to open '%s' for sync: [2] No such file or directory", strPtr(pathName));
TEST_RESULT_VOID(storagePathSyncP(storageTest, pathName, .ignoreMissing = true), "ignore missing path");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storagePathCreateNP(storageTest, pathName), "create path to sync");
TEST_RESULT_VOID(storagePathSyncNP(storageTest, pathName), "sync path");
}
// *****************************************************************************************************************************
if (testBegin("storageOpenWrite()"))
if (testBegin("storageNewRead()"))
{
TEST_ERROR(
storageOpenWriteNP(storageTest, strNew(testPath())), FileOpenError,
strPtr(strNewFmt("unable to open '%s' for write: [21] Is a directory", testPath())));
StorageFileRead *file = NULL;
String *fileName = strNewFmt("%s/readtest.txt", testPath());
TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new read file (defaults)");
TEST_ERROR_FMT(
storageFileReadOpen(file), FileOpenError,
"unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
// -------------------------------------------------------------------------------------------------------------------------
StorageFile *file = NULL;
String *fileName = strNewFmt("%s/testfile", testPath());
TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileName)))), 0, "create read file");
TEST_ASSIGN(file, storageOpenWriteNP(storageTest, fileName), "open file for write (defaults)");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, "check file mode");
close(STORAGE_DATA(file)->handle);
TEST_RESULT_BOOL(storageFileReadOpen(file), true, " open file");
TEST_RESULT_VOID(storageFileReadClose(file), " close file");
}
storageRemoveP(storageTest, fileName, .errorOnMissing = true);
// *****************************************************************************************************************************
if (testBegin("storageNewWrite()"))
{
StorageFileWrite *file = NULL;
TEST_ASSIGN(file, storageNewWriteP(storageTest, fileNoPerm, .noAtomic = true), "new write file (defaults)");
TEST_ERROR_FMT(
storageFileWriteOpen(file), FileOpenError,
"unable to open '%s' for write: [13] Permission denied", strPtr(fileNoPerm));
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(file, storageOpenWriteP(storageTest, fileName, .mode = 0777), "open file for write (custom)");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0777, "check file mode");
close(STORAGE_DATA(file)->handle);
String *fileName = strNewFmt("%s/sub1/testfile", testPath());
storageRemoveP(storageTest, fileName, .errorOnMissing = true);
TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file (defaults)");
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
TEST_RESULT_VOID(storageFileWriteClose(file), " close file");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0750, " check path mode");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, " check file mode");
// -------------------------------------------------------------------------------------------------------------------------
fileName = strNewFmt("%s/sub2/testfile", testPath());
TEST_ASSIGN(
file, storageNewWriteP(storageTest, fileName, .modePath = 0700, .modeFile = 0600), "new write file (set mode)");
TEST_RESULT_VOID(storageFileWriteOpen(file), " open file");
TEST_RESULT_VOID(storageFileWriteClose(file), " close file");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0700, " check path mode");
TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0600, " check file mode");
}
// *****************************************************************************************************************************
@ -323,42 +465,32 @@ testRun()
{
Storage *storageTest = storageNewP(strNew("/"), .write = true);
TEST_ERROR(
storageGetNP(storageOpenReadNP(storageTest, strNew(testPath()))), FileReadError,
strPtr(strNewFmt("unable to read '%s': [21] Is a directory", testPath())));
TEST_ERROR_FMT(
storageGetNP(storageNewReadNP(storageTest, strNew(testPath()))), FileReadError,
"unable to read '%s': [21] Is a directory", testPath());
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storagePutNP(storageOpenWriteNP(storageTest, strNewFmt("%s/test.empty", testPath())), NULL), "put empty file");
String *emptyFile = strNewFmt("%s/test.empty", testPath());
TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, emptyFile), NULL), "put empty file");
TEST_RESULT_BOOL(storageExistsNP(storageTest, emptyFile), true, "check empty file exists");
// -------------------------------------------------------------------------------------------------------------------------
Buffer *buffer = bufNewStr(strNew("TESTFILE\n"));
StorageFile *file = NULL;
StorageFileDataPosix fileData = {.handle = 999999};
MEM_CONTEXT_NEW_BEGIN("FileTest")
{
file = storageFileNew(storageTest, strNew("badfile"), storageFileTypeWrite, &fileData);
}
MEM_CONTEXT_NEW_END();
TEST_ERROR(storagePutNP(file, buffer), FileWriteError, "unable to write 'badfile': [9] Bad file descriptor");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(
storagePutNP(storageOpenWriteNP(storageTest, strNewFmt("%s/test.txt", testPath())), buffer), "put text file");
storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/test.txt", testPath())), buffer), "put text file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_PTR(
storageGetNP(storageOpenReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true)),
storageGetNP(storageNewReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true)),
NULL, "get missing file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.empty", testPath()))), "get empty");
TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.empty", testPath()))), "get empty");
TEST_RESULT_INT(bufSize(buffer), 0, "size is 0");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
TEST_RESULT_INT(bufSize(buffer), 9, "check size");
TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TESTFILE\n", bufSize(buffer)) == 0, true, "check content");
@ -366,7 +498,7 @@ testRun()
const Storage *storage = storageTest;
((Storage *)storage)->bufferSize = 2;
TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
TEST_RESULT_INT(bufSize(buffer), 9, "check size");
TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TESTFILE\n", bufSize(buffer)) == 0, true, "check content");
}
@ -376,9 +508,9 @@ testRun()
{
// -------------------------------------------------------------------------------------------------------------------------
TEST_RESULT_VOID(storageRemoveNP(storageTest, strNew("missing")), "remove missing file");
TEST_ERROR(
TEST_ERROR_FMT(
storageRemoveP(storageTest, strNew("missing"), .errorOnMissing = true), FileRemoveError,
strPtr(strNewFmt("unable to remove '%s/missing': [2] No such file or directory", testPath())));
"unable to remove '%s/missing': [2] No such file or directory", testPath());
// -------------------------------------------------------------------------------------------------------------------------
String *fileExists = strNewFmt("%s/exists", testPath());
@ -387,8 +519,8 @@ testRun()
TEST_RESULT_VOID(storageRemoveNP(storageTest, fileExists), "remove exists file");
// -------------------------------------------------------------------------------------------------------------------------
TEST_ERROR(
TEST_ERROR_FMT(
storageRemoveNP(storageTest, fileNoPerm), FileRemoveError,
strPtr(strNewFmt("unable to remove '%s': [13] Permission denied", strPtr(fileNoPerm))));
"unable to remove '%s': [13] Permission denied", strPtr(fileNoPerm));
}
}