Storage object improvements.

* Add storageCopy(), storageMove(), and storagePathSync().
* Separate StorageFile object into separate read and write objects.
* Abstract out Posix file read/write objects.

build/error.yaml

@@ -80,6 +80,9 @@ memory: 94
 cipher: 95
 param-invalid: 96
 
+# Unable to close a path
+path-close: 97
+
 # This error should not be thrown directly -- it serves as a parent for the C errors
 runtime: 122
 

doc/xml/release.xml

@@ -62,7 +62,7 @@
                     </release-item>
 
                     <release-item>
-                        <p>Storage object improvements.  Convert all functions to variadic functions.  Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function.  Add <code>storageExists()</code>, <code>storagePathCreate()</code>, and <code>storageRemove()</code>.  Add <code>StorageFile</code> object and <code>storageOpenRead()</code>/<code>storageOpenWrite()</code>.  Abstract Posix driver code into a separate module.  Add <code>storagePathRemove()</code> and use it in the Perl Posix driver.</p>
+                        <p>Storage object improvements.  Convert all functions to variadic functions.  Enforce read-only storage. Add <code>storageLocalWrite()</code> helper function.  Add <code>storageCopy()</code>, <code>storageExists()</code>, <code>storageMove()</code>, <code>storageNewRead()</code>/<code>storageNewWrite()</code>, <code>storagePathCreate()</code>, <code>storagePathRemove()</code>, <code>storagePathSync()</code>, and <code>storageRemove()</code>.  Add <code>StorageFileRead</code> and <code>StorageFileWrite</code> objects.  Abstract Posix driver code into a separate module. Call <code>storagePathCreate()</code> from the Perl Posix driver.</p>
                     </release-item>
 
                     <release-item>

lib/pgBackRest/Common/ExceptionAuto.pm

@@ -161,6 +161,8 @@ use constant ERROR_CIPHER                                           => 95;
 push @EXPORT, qw(ERROR_CIPHER);
 use constant ERROR_PARAM_INVALID                                    => 96;
 push @EXPORT, qw(ERROR_PARAM_INVALID);
+use constant ERROR_PATH_CLOSE                                       => 97;
+push @EXPORT, qw(ERROR_PATH_CLOSE);
 use constant ERROR_RUNTIME                                          => 122;
 push @EXPORT, qw(ERROR_RUNTIME);
 use constant ERROR_INVALID                                          => 123;

libc/LibC.xs

@@ -50,7 +50,7 @@ These includes are from the src directory.  There is no Perl-specific code in th
 #include "config/parse.h"
 #include "perl/config.h"
 #include "postgres/pageChecksum.h"
-#include "storage/driver/posix.h"
+#include "storage/driver/posix/driver.h"
 
 /***********************************************************************************************************************************
 Helper macros

libc/Makefile.PL

@@ -102,8 +102,12 @@ my @stryCFile =
     'config/parse.c',
     'perl/config.c',
     'postgres/pageChecksum.c',
-    'storage/driver/posix.c',
+    'storage/driver/posix/driver.c',
-    'storage/file.c',
+    'storage/driver/posix/driverFile.c',
+    'storage/driver/posix/driverRead.c',
+    'storage/driver/posix/driverWrite.c',
+    'storage/fileRead.c',
+    'storage/fileWrite.c',
     'storage/helper.c',
     'storage/storage.c',
 );

src/Makefile

@@ -81,8 +81,12 @@ SRCS = \
 	config/parse.c \
 	perl/config.c \
 	perl/exec.c \
-	storage/driver/posix.c \
+	storage/driver/posix/driver.c \
-	storage/file.c \
+	storage/driver/posix/driverFile.c \
+	storage/driver/posix/driverRead.c \
+	storage/driver/posix/driverWrite.c \
+	storage/fileRead.c \
+	storage/fileWrite.c \
 	storage/helper.c \
 	storage/storage.c \
 	main.c

src/command/archive/push/push.c

@@ -49,7 +49,7 @@ walStatus(const String *walSegment, bool confessOnError)
             const String *statusFile = strLstGet(fileList, 0);
 
             String *content = strNewBuf(
-                storageGetNP(storageOpenReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
+                storageGetNP(storageNewReadNP(storageSpool(), strNewFmt("%s/%s", STORAGE_SPOOL_ARCHIVE_OUT, strPtr(statusFile)))));
 
             // Get the code and message if the file has content
             int code = 0;

src/common/error.auto.c

@@ -78,6 +78,7 @@ ERROR_DEFINE( 93, FileExistsError, RuntimeError);
 ERROR_DEFINE( 94, MemoryError, RuntimeError);
 ERROR_DEFINE( 95, CipherError, RuntimeError);
 ERROR_DEFINE( 96, ParamInvalidError, RuntimeError);
+ERROR_DEFINE( 97, PathCloseError, RuntimeError);
 ERROR_DEFINE(122, RuntimeError, RuntimeError);
 ERROR_DEFINE(123, InvalidError, RuntimeError);
 ERROR_DEFINE(124, UnhandledError, RuntimeError);
@@ -159,6 +160,7 @@ static const ErrorType *errorTypeList[] =
     &MemoryError,
     &CipherError,
     &ParamInvalidError,
+    &PathCloseError,
     &RuntimeError,
     &InvalidError,
     &UnhandledError,

src/common/error.auto.h

@@ -80,6 +80,7 @@ ERROR_DECLARE(FileExistsError);
 ERROR_DECLARE(MemoryError);
 ERROR_DECLARE(CipherError);
 ERROR_DECLARE(ParamInvalidError);
+ERROR_DECLARE(PathCloseError);
 ERROR_DECLARE(RuntimeError);
 ERROR_DECLARE(InvalidError);
 ERROR_DECLARE(UnhandledError);

src/common/ini.c

@@ -212,7 +212,7 @@ iniLoad(Ini *this, const String *fileName)
 
         MEM_CONTEXT_TEMP_BEGIN()
         {
-            iniParse(this, strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), this->fileName))));
+            iniParse(this, strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), this->fileName))));
         }
         MEM_CONTEXT_TEMP_END();
     }

src/config/parse.c

@@ -164,7 +164,7 @@ cfgFileLoad(                                                        // NOTE: Pas
             configFileName = optConfigDefault;
 
         // Load the config file
-        Buffer *buffer = storageGetNP(storageOpenReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
+        Buffer *buffer = storageGetNP(storageNewReadP(storageLocal(), configFileName, .ignoreMissing = !configRequired));
 
         // Convert the contents of the file buffer to the config string object
         if (buffer != NULL)
@@ -172,7 +172,7 @@ cfgFileLoad(                                                        // NOTE: Pas
         else if (strEq(configFileName, optConfigDefaultCurrent))
         {
             // If confg is current default and it was not found, attempt to load the config file from the old default location
-            buffer = storageGetNP(storageOpenReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
+            buffer = storageGetNP(storageNewReadP(storageLocal(), origConfigDefault, .ignoreMissing = !configRequired));
 
             if (buffer != NULL)
                 result = strNewBuf(buffer);
@@ -210,7 +210,7 @@ cfgFileLoad(                                                        // NOTE: Pas
             for (unsigned int listIdx = 0; listIdx < strLstSize(list); listIdx++)
             {
                 Buffer *fileBuffer = storageGetNP(
-                    storageOpenReadP(
+                    storageNewReadP(
                         storageLocal(), strNewFmt("%s/%s", strPtr(configIncludePath), strPtr(strLstGet(list, listIdx))),
                         .ignoreMissing = true));
 

src/storage/driver/posix/driver.c

@@ -1,29 +1,19 @@
 /***********************************************************************************************************************************
-Storage Posix Driver
+Storage Driver Posix
 ***********************************************************************************************************************************/
 #include <dirent.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <stdio.h>
 #include <sys/stat.h>
 #include <unistd.h>
 
 #include "common/memContext.h"
 #include "common/regExp.h"
-#include "storage/driver/posix.h"
+#include "storage/driver/posix/driver.h"
+#include "storage/driver/posix/driverFile.h"
 #include "storage/storage.h"
 
-/***********************************************************************************************************************************
-Storage file data - holds the file handle.
-***********************************************************************************************************************************/
-typedef struct StorageFileDataPosix
-{
-    MemContext *memContext;
-    int handle;
-} StorageFileDataPosix;
-
-#define STORAGE_DATA(file)                                                                                                         \
-    ((StorageFileDataPosix *)storageFileData(file))
-
 /***********************************************************************************************************************************
 Does a file/path exist?
 ***********************************************************************************************************************************/
@@ -48,61 +38,6 @@ storageDriverPosixExists(const String *path)
     return result;
 }
 
-/***********************************************************************************************************************************
-Read from storage into a buffer
-***********************************************************************************************************************************/
-Buffer *
-storageDriverPosixGet(const StorageFile *file)
-{
-    Buffer *result = NULL;
-
-    MEM_CONTEXT_TEMP_BEGIN()
-    {
-        TRY_BEGIN()
-        {
-            size_t bufferSize = storageBufferSize(storageFileStorage(file));
-            result = bufNew(bufferSize);
-
-            // Create result buffer with buffer size
-            ssize_t actualBytes = 0;
-            size_t totalBytes = 0;
-
-            do
-            {
-                // Grow the buffer on subsequent reads
-                if (totalBytes != 0)
-                    bufResize(result, bufSize(result) + bufferSize);
-
-                // Read and handle errors
-                actualBytes = read(
-                    STORAGE_DATA(file)->handle, bufPtr(result) + totalBytes, bufferSize);
-
-                // Error occurred during write
-                if (actualBytes == -1)
-                    THROW_SYS_ERROR(FileReadError, "unable to read '%s'", strPtr(storageFileName(file)));
-
-                // Track total bytes read
-                totalBytes += (size_t)actualBytes;
-            }
-            while (actualBytes != 0);
-
-            // Resize buffer to total bytes read
-            bufResize(result, totalBytes);
-        }
-        FINALLY()
-        {
-            close(STORAGE_DATA(file)->handle);
-            storageFileFree(file);
-        }
-        TRY_END();
-
-        bufMove(result, MEM_CONTEXT_OLD());
-    }
-    MEM_CONTEXT_TEMP_END();
-
-    return result;
-}
-
 /***********************************************************************************************************************************
 Get a list of files from a directory
 ***********************************************************************************************************************************/
@@ -166,59 +101,59 @@ storageDriverPosixList(const String *path, bool errorOnMissing, const String *ex
 }
 
 /***********************************************************************************************************************************
-Open a file for reading
+Move a file
 ***********************************************************************************************************************************/
-void *
+bool
-storageDriverPosixOpenRead(const String *file, bool ignoreMissing)
+storageDriverPosixMove(StorageFileReadPosix *source, StorageFileWritePosix *destination)
 {
-    StorageFileDataPosix *result = NULL;
+    bool result = true;
 
-    // Open the file and handle errors
+    MEM_CONTEXT_TEMP_BEGIN()
-    int fileHandle = open(strPtr(file), O_RDONLY, 0);
+    {
+        const String *sourceFile = storageFileReadPosixName(source);
+        const String *destinationFile = storageFileWritePosixName(destination);
+        const String *destinationPath = storageFileWritePosixPath(destination);
 
-    if (fileHandle == -1)
+        // Attempt to move the file
-    {
+        if (rename(strPtr(sourceFile), strPtr(destinationFile)) == -1)
-        // Error unless ignore missing is specified
-        if (!ignoreMissing || errno != ENOENT)
-            THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for read", strPtr(file));
-    }
-    // Else create the storage file and data
-    else
-    {
-        MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
         {
-            result = memNew(sizeof(StorageFileDataPosix));
+            // Detemine which file/path is missing
-            result->memContext = MEM_CONTEXT_NEW();
+            if (errno == ENOENT)
-            result->handle = fileHandle;
+            {
+                if (!storageDriverPosixExists(sourceFile))
+                    THROW_SYS_ERROR(FileMissingError, "unable to move missing file '%s'", strPtr(sourceFile));
+
+                if (!storageFileWritePosixCreatePath(destination))
+                {
+                    THROW_SYS_ERROR(
+                        PathMissingError, "unable to move '%s' to missing path '%s'", strPtr(sourceFile), strPtr(destinationPath));
+                }
+
+                storageDriverPosixPathCreate(destinationPath, false, false, storageFileWritePosixModePath(destination));
+                result = storageDriverPosixMove(source, destination);
+            }
+            // Else the destination is on a different device so a copy will be needed
+            else if (errno == EXDEV)
+            {
+                result = false;
+            }
+            else
+                THROW_SYS_ERROR(FileMoveError, "unable to move '%s' to '%s'", strPtr(sourceFile), strPtr(destinationFile));
+        }
+        // Sync paths on success
+        else
+        {
+            // Sync source path if the destination path was synced and the paths are not equal
+            if (storageFileWritePosixSyncPath(destination))
+            {
+                String *sourcePath = strPath(sourceFile);
+
+                if (!strEq(destinationPath, sourcePath))
+                    storageDriverPosixPathSync(sourcePath, false);
+            }
         }
-        MEM_CONTEXT_NEW_END();
     }
-
+    MEM_CONTEXT_TEMP_END();
-    return result;
-}
-
-/***********************************************************************************************************************************
-Open a file for writing
-***********************************************************************************************************************************/
-void *
-storageDriverPosixOpenWrite(const String *file, mode_t mode)
-{
-    // Open the file and handle errors
-    int fileHandle = open(strPtr(file), O_CREAT | O_TRUNC | O_WRONLY, mode);
-
-    if (fileHandle == -1)
-        THROW_SYS_ERROR(FileOpenError, "unable to open '%s' for write", strPtr(file));
-
-    // Create the storage file and data
-    StorageFileDataPosix *result = NULL;
-
-    MEM_CONTEXT_NEW_BEGIN("StorageFileDataPosix")
-    {
-        result = memNew(sizeof(StorageFileDataPosix));
-        result->memContext = MEM_CONTEXT_NEW();
-        result->handle = fileHandle;
-    }
-    MEM_CONTEXT_NEW_END();
 
     return result;
 }
@@ -291,22 +226,23 @@ storageDriverPosixPathRemove(const String *path, bool errorOnMissing, bool recur
 }
 
 /***********************************************************************************************************************************
-Write a buffer to storage
+Sync a path
 ***********************************************************************************************************************************/
 void
-storageDriverPosixPut(const StorageFile *file, const Buffer *buffer)
+storageDriverPosixPathSync(const String *path, bool ignoreMissing)
 {
-    TRY_BEGIN()
+    // Open directory and handle errors
+    int handle = storageFilePosixOpen(path, O_RDONLY, 0, ignoreMissing, &PathOpenError, "sync");
+
+    // On success
+    if (handle != -1)
     {
-        if (write(STORAGE_DATA(file)->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
+        // Attempt to sync the directory
-            THROW_SYS_ERROR(FileWriteError, "unable to write '%s'", strPtr(storageFileName(file)));
+        storageFilePosixSync(handle, path, &PathSyncError, true);
+
+        // Close the directory
+        storageFilePosixClose(handle, path, &PathCloseError);
     }
-    FINALLY()
-    {
-        close(STORAGE_DATA(file)->handle);
-        storageFileFree(file);
-    }
-    TRY_END();
 }
 
 /***********************************************************************************************************************************

src/storage/driver/posix/driver.h

@@ -1,27 +1,25 @@
 /***********************************************************************************************************************************
-Storage Posix Driver
+Storage Driver Posix
 ***********************************************************************************************************************************/
-#ifndef STORAGE_DRIVER_POSIX_H
+#ifndef STORAGE_DRIVER_POSIX_DRIVER_H
-#define STORAGE_DRIVER_POSIX_H
+#define STORAGE_DRIVER_POSIX_DRIVER_H
 
 #include <sys/types.h>
 
 #include "common/type/buffer.h"
-#include "common/type/string.h"
+#include "common/type/stringList.h"
-#include "storage/file.h"
+#include "storage/driver/posix/driverRead.h"
-#include "storage/storage.h"
+#include "storage/driver/posix/driverWrite.h"
 
 /***********************************************************************************************************************************
 Function
 ***********************************************************************************************************************************/
 bool storageDriverPosixExists(const String *path);
-Buffer *storageDriverPosixGet(const StorageFile *file);
 StringList *storageDriverPosixList(const String *path, bool errorOnMissing, const String *expression);
-void *storageDriverPosixOpenRead(const String *file, bool ignoreMissing);
+bool storageDriverPosixMove(StorageFileReadPosix *source, StorageFileWritePosix *destination);
-void *storageDriverPosixOpenWrite(const String *file, mode_t mode);
 void storageDriverPosixPathCreate(const String *path, bool errorOnExists, bool noParentCreate, mode_t mode);
 void storageDriverPosixPathRemove(const String *path, bool errorOnMissing, bool recurse);
-void storageDriverPosixPut(const StorageFile *file, const Buffer *buffer);
+void storageDriverPosixPathSync(const String *path, bool ignoreMissing);
 void storageDriverPosixRemove(const String *file, bool errorOnMissing);
 
 #endif

src/storage/driver/posix/driverFile.c

@@ -0,0 +1,63 @@
+/***********************************************************************************************************************************
+Storage File Routines For Posix
+***********************************************************************************************************************************/
+// So fsync() will work on older glib versions
+#ifndef _POSIX_C_SOURCE
+    #define _POSIX_C_SOURCE 200112L
+#endif
+
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "common/assert.h"
+#include "storage/driver/posix/driverFile.h"
+
+/***********************************************************************************************************************************
+Open a file
+
+Returns the handle of the open file, or -1 for reads if the file is missing and -1 for writes if the path is mssing.
+***********************************************************************************************************************************/
+int
+storageFilePosixOpen(
+    const String *name, int flags, mode_t mode, bool ignoreMissing, const ErrorType *errorType, const char *purpose)
+{
+    int result = -1;
+
+    result = open(strPtr(name), flags, mode);
+
+    if (result == -1)
+    {
+        if (errno != ENOENT || !ignoreMissing)
+            THROWP_SYS_ERROR(errorType, "unable to open '%s' for %s", strPtr(name), purpose);
+    }
+
+    return result;
+}
+
+/***********************************************************************************************************************************
+Sync a file/directory handle
+***********************************************************************************************************************************/
+void
+storageFilePosixSync(int handle, const String *name, const ErrorType *errorType, bool closeOnError)
+{
+    if (fsync(handle) == -1)
+    {
+        int errNo = errno;
+
+        // Close if requested but don't report errors -- we want to report the sync error instead
+        if (closeOnError)
+            close(handle);
+
+        THROWP_SYS_ERROR_CODE(errNo, errorType, "unable to sync '%s'", strPtr(name));
+    }
+}
+
+/***********************************************************************************************************************************
+Close a file/directory handle
+***********************************************************************************************************************************/
+void
+storageFilePosixClose(int handle, const String *name, const ErrorType *errorType)
+{
+    if (close(handle) == -1)
+        THROWP_SYS_ERROR(errorType, "unable to close '%s'", strPtr(name));
+}

src/storage/driver/posix/driverFile.h

@@ -0,0 +1,18 @@
+/***********************************************************************************************************************************
+Storage File Routines For Posix
+***********************************************************************************************************************************/
+#ifndef STORAGE_DRIVER_POSIX_DRIVERFILE_H
+#define STORAGE_DRIVER_POSIX_DRIVERFILE_H
+
+#include "common/error.h"
+#include "common/type/string.h"
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+int storageFilePosixOpen(
+    const String *name, int flags, mode_t mode, bool ignoreMissing, const ErrorType *errorType, const char *purpose);
+void storageFilePosixSync(int handle, const String *name, const ErrorType *errorType, bool closeOnError);
+void storageFilePosixClose(int handle, const String *name, const ErrorType *errorType);
+
+#endif

src/storage/driver/posix/driverRead.c

@@ -0,0 +1,165 @@
+/***********************************************************************************************************************************
+Storage File Read Driver For Posix
+***********************************************************************************************************************************/
+#include <fcntl.h>
+#include <unistd.h>
+
+#include "common/assert.h"
+#include "common/memContext.h"
+#include "storage/driver/posix/driverFile.h"
+#include "storage/driver/posix/driverRead.h"
+
+/***********************************************************************************************************************************
+Storage file structure
+***********************************************************************************************************************************/
+struct StorageFileReadPosix
+{
+    MemContext *memContext;
+
+    String *name;
+    bool ignoreMissing;
+    size_t bufferSize;
+
+    int handle;
+    bool eof;
+};
+
+/***********************************************************************************************************************************
+Create a new file
+***********************************************************************************************************************************/
+StorageFileReadPosix *
+storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize)
+{
+    StorageFileReadPosix *this = NULL;
+
+    ASSERT_DEBUG(name != NULL);
+    ASSERT_DEBUG(bufferSize > 0);
+
+    // Create the file object
+    MEM_CONTEXT_NEW_BEGIN("StorageFileReadPosix")
+    {
+        this = memNew(sizeof(StorageFileReadPosix));
+        this->memContext = MEM_CONTEXT_NEW();
+        this->name = strDup(name);
+        this->ignoreMissing = ignoreMissing;
+        this->bufferSize = bufferSize;
+
+        this->handle = -1;
+    }
+    MEM_CONTEXT_NEW_END();
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Open the file
+***********************************************************************************************************************************/
+bool
+storageFileReadPosixOpen(StorageFileReadPosix *this)
+{
+    bool result = false;
+
+    ASSERT_DEBUG(this != NULL);
+    ASSERT_DEBUG(this->handle == -1);
+
+    // Open the file and handle errors
+    this->handle = storageFilePosixOpen(this->name, O_RDONLY, 0, this->ignoreMissing, &FileOpenError, "read");
+
+    // On success set free callback to ensure file handle is freed
+    if (this->handle != -1)
+    {
+        memContextCallback(this->memContext, (MemContextCallback)storageFileReadPosixFree, this);
+        result = true;
+    }
+
+    return result;
+}
+
+/***********************************************************************************************************************************
+Read from a file
+***********************************************************************************************************************************/
+Buffer *
+storageFileReadPosix(StorageFileReadPosix *this)
+{
+    Buffer *result = NULL;
+
+    ASSERT_DEBUG(this != NULL);
+
+    // Read if EOF has not been reached
+    if (!this->eof)
+    {
+        result = bufNew(this->bufferSize);
+
+        // Read and handle errors
+        ssize_t actualBytes = read(this->handle, bufPtr(result), this->bufferSize);
+
+        // Error occurred during write
+        if (actualBytes == -1)
+            THROW_SYS_ERROR(FileReadError, "unable to read '%s'", strPtr(this->name));
+
+        // If no data was read then free the buffer and mark the file as EOF
+        if (actualBytes == 0)
+        {
+            this->eof = true;
+            bufFree(result);
+            result = NULL;
+        }
+        else
+            bufResize(result, (size_t)actualBytes);
+    }
+
+    return result;
+}
+
+/***********************************************************************************************************************************
+Close the file
+***********************************************************************************************************************************/
+void
+storageFileReadPosixClose(StorageFileReadPosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    // Close if the file has not already been closed
+    if (this->handle != -1)
+    {
+        // Close the file
+        storageFilePosixClose(this->handle, this->name, &FileCloseError);
+
+        this->handle = -1;
+    }
+}
+
+/***********************************************************************************************************************************
+Should a missing file be ignored?
+***********************************************************************************************************************************/
+bool
+storageFileReadPosixIgnoreMissing(StorageFileReadPosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->ignoreMissing;
+}
+
+/***********************************************************************************************************************************
+File name
+***********************************************************************************************************************************/
+const String *
+storageFileReadPosixName(StorageFileReadPosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->name;
+}
+
+/***********************************************************************************************************************************
+Free the file
+***********************************************************************************************************************************/
+void
+storageFileReadPosixFree(StorageFileReadPosix *this)
+{
+    if (this != NULL)
+    {
+        storageFileReadPosixClose(this);
+        memContextFree(this->memContext);
+    }
+}

src/storage/driver/posix/driverRead.h

@@ -0,0 +1,38 @@
+/***********************************************************************************************************************************
+Storage File Read Driver For Posix
+***********************************************************************************************************************************/
+#ifndef STORAGE_DRIVER_POSIX_DRIVERREAD_H
+#define STORAGE_DRIVER_POSIX_DRIVERREAD_H
+
+/***********************************************************************************************************************************
+Read file object
+***********************************************************************************************************************************/
+typedef struct StorageFileReadPosix StorageFileReadPosix;
+
+#include "common/type/buffer.h"
+#include "common/type/string.h"
+
+/***********************************************************************************************************************************
+Constructor
+***********************************************************************************************************************************/
+StorageFileReadPosix *storageFileReadPosixNew(const String *name, bool ignoreMissing, size_t bufferSize);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+bool storageFileReadPosixOpen(StorageFileReadPosix *this);
+Buffer *storageFileReadPosix(StorageFileReadPosix *this);
+void storageFileReadPosixClose(StorageFileReadPosix *this);
+
+/***********************************************************************************************************************************
+Getters
+***********************************************************************************************************************************/
+bool storageFileReadPosixIgnoreMissing(StorageFileReadPosix *this);
+const String *storageFileReadPosixName(StorageFileReadPosix *this);
+
+/***********************************************************************************************************************************
+Destructor
+***********************************************************************************************************************************/
+void storageFileReadPosixFree(StorageFileReadPosix *this);
+
+#endif

src/storage/driver/posix/driverWrite.c

@@ -0,0 +1,255 @@
+/***********************************************************************************************************************************
+Storage File Write Driver For Posix
+***********************************************************************************************************************************/
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include "common/assert.h"
+#include "common/memContext.h"
+#include "storage/driver/posix/driverFile.h"
+#include "storage/driver/posix/driverWrite.h"
+#include "storage/driver/posix/driver.h"
+#include "storage/fileWrite.h"
+
+/***********************************************************************************************************************************
+Storage file structure
+***********************************************************************************************************************************/
+struct StorageFileWritePosix
+{
+    MemContext *memContext;
+
+    String *path;
+    String *name;
+    String *nameTmp;
+    mode_t modeFile;
+    mode_t modePath;
+    bool noCreatePath;
+    bool noSyncFile;
+    bool noSyncPath;
+    bool noAtomic;
+
+    int handle;
+};
+
+/***********************************************************************************************************************************
+File open constants
+
+Since open is called more than once use constants to make sure these parameters are always the same
+***********************************************************************************************************************************/
+#define FILE_OPEN_FLAGS                                             (O_CREAT | O_TRUNC | O_WRONLY)
+#define FILE_OPEN_PURPOSE                                           "write"
+#define FILE_OPEN_ERROR                                             &FileOpenError
+
+/***********************************************************************************************************************************
+Create a new file
+***********************************************************************************************************************************/
+StorageFileWritePosix *
+storageFileWritePosixNew(
+    const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic)
+{
+    StorageFileWritePosix *this = NULL;
+
+    ASSERT_DEBUG(name != NULL);
+
+    // Create the file
+    MEM_CONTEXT_NEW_BEGIN("StorageFileWritePosix")
+    {
+        this = memNew(sizeof(StorageFileWritePosix));
+        this->memContext = MEM_CONTEXT_NEW();
+        this->path = strPath(name);
+        this->name = strDup(name);
+        this->nameTmp = noAtomic ? this->name : strNewFmt("%s." STORAGE_FILE_TEMP_EXT, strPtr(name));
+        this->modeFile = modeFile;
+        this->modePath = modePath;
+        this->noCreatePath = noCreatePath;
+        this->noSyncFile = noSyncFile;
+        this->noSyncPath = noSyncPath;
+        this->noAtomic = noAtomic;
+
+        this->handle = -1;
+    }
+    MEM_CONTEXT_NEW_END();
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Open the file
+***********************************************************************************************************************************/
+void
+storageFileWritePosixOpen(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+    ASSERT_DEBUG(this->handle == -1);
+
+    // Open the file and handle errors
+    this->handle = storageFilePosixOpen(
+        this->nameTmp, FILE_OPEN_FLAGS, this->modeFile, !this->noCreatePath, FILE_OPEN_ERROR, FILE_OPEN_PURPOSE);
+
+    // If path is missing
+    if (this->handle == -1)
+    {
+        // Create the path
+        storageDriverPosixPathCreate(this->path, false, false, this->modePath);
+
+        // Try the open again
+        this->handle = storageFilePosixOpen(
+            this->nameTmp, FILE_OPEN_FLAGS, this->modeFile, false, FILE_OPEN_ERROR, FILE_OPEN_PURPOSE);
+    }
+    // On success set free callback to ensure file handle is freed
+    else
+        memContextCallback(this->memContext, (MemContextCallback)storageFileWritePosixFree, this);
+}
+
+/***********************************************************************************************************************************
+Write to a file
+***********************************************************************************************************************************/
+void
+storageFileWritePosix(StorageFileWritePosix *this, const Buffer *buffer)
+{
+    ASSERT_DEBUG(this != NULL);
+    ASSERT_DEBUG(buffer != NULL);
+    ASSERT_DEBUG(this->handle != -1);
+
+    // Write the data
+    if (write(this->handle, bufPtr(buffer), bufSize(buffer)) != (ssize_t)bufSize(buffer))
+        THROW_SYS_ERROR(FileWriteError, "unable to write '%s'", strPtr(this->name));
+}
+
+/***********************************************************************************************************************************
+Close the file
+***********************************************************************************************************************************/
+void
+storageFileWritePosixClose(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    // Close if the file has not already been closed
+    if (this->handle != -1)
+    {
+        // Sync the file
+        if (!this->noSyncFile)
+            storageFilePosixSync(this->handle, this->name, &FileSyncError, false);
+
+        // Close the file
+        storageFilePosixClose(this->handle, this->name, &FileCloseError);
+
+        // Rename from temp file
+        if (!this->noAtomic)
+        {
+            if (rename(strPtr(this->nameTmp), strPtr(this->name)) == -1)
+                THROW_SYS_ERROR(FileMoveError, "unable to move '%s' to '%s'", strPtr(this->nameTmp), strPtr(this->name));
+        }
+
+        // Sync the path
+        if (!this->noSyncPath)
+            storageDriverPosixPathSync(this->path, false);
+
+        // This marks the file as closed
+        this->handle = -1;
+    }
+}
+
+/***********************************************************************************************************************************
+Will the file be written atomically?
+
+For the posix driver this means writing to a temp file first and then renaming once it is closed and synced.
+***********************************************************************************************************************************/
+bool
+storageFileWritePosixAtomic(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return !this->noAtomic;
+}
+
+/***********************************************************************************************************************************
+Will the path be created for the file if it does not exist?
+***********************************************************************************************************************************/
+bool
+storageFileWritePosixCreatePath(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return !this->noCreatePath;
+}
+
+/***********************************************************************************************************************************
+Mode for the file to be created
+***********************************************************************************************************************************/
+mode_t
+storageFileWritePosixModeFile(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->modeFile;
+}
+
+/***********************************************************************************************************************************
+Mode for any paths that are created while writing the file
+***********************************************************************************************************************************/
+mode_t
+storageFileWritePosixModePath(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->modePath;
+}
+
+/***********************************************************************************************************************************
+File name
+***********************************************************************************************************************************/
+const String *
+storageFileWritePosixName(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->name;
+}
+
+/***********************************************************************************************************************************
+File path
+***********************************************************************************************************************************/
+const String *
+storageFileWritePosixPath(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->path;
+}
+
+/***********************************************************************************************************************************
+Will the file be synced after it is closed?
+***********************************************************************************************************************************/
+bool
+storageFileWritePosixSyncFile(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return !this->noSyncFile;
+}
+
+/***********************************************************************************************************************************
+Will the directory be synced to disk after the write is completed?
+***********************************************************************************************************************************/
+bool
+storageFileWritePosixSyncPath(StorageFileWritePosix *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return !this->noSyncPath;
+}
+
+/***********************************************************************************************************************************
+Free the file
+***********************************************************************************************************************************/
+void
+storageFileWritePosixFree(StorageFileWritePosix *this)
+{
+    if (this != NULL)
+    {
+        storageFileWritePosixClose(this);
+        memContextFree(this->memContext);
+    }
+}

src/storage/driver/posix/driverWrite.h

@@ -0,0 +1,47 @@
+/***********************************************************************************************************************************
+Storage File Write Driver For Posix
+***********************************************************************************************************************************/
+#ifndef STORAGE_DRIVER_POSIX_DRIVERWRITE_H
+#define STORAGE_DRIVER_POSIX_DRIVERWRITE_H
+
+#include <sys/types.h>
+
+/***********************************************************************************************************************************
+Write file object
+***********************************************************************************************************************************/
+typedef struct StorageFileWritePosix StorageFileWritePosix;
+
+#include "common/type/buffer.h"
+#include "common/type/string.h"
+
+/***********************************************************************************************************************************
+Constructor
+***********************************************************************************************************************************/
+StorageFileWritePosix *storageFileWritePosixNew(
+    const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+void storageFileWritePosixOpen(StorageFileWritePosix *this);
+void storageFileWritePosix(StorageFileWritePosix *this, const Buffer *buffer);
+void storageFileWritePosixClose(StorageFileWritePosix *this);
+
+/***********************************************************************************************************************************
+Getters
+***********************************************************************************************************************************/
+bool storageFileWritePosixAtomic(StorageFileWritePosix *this);
+bool storageFileWritePosixCreatePath(StorageFileWritePosix *this);
+mode_t storageFileWritePosixModeFile(StorageFileWritePosix *this);
+mode_t storageFileWritePosixModePath(StorageFileWritePosix *this);
+const String *storageFileWritePosixName(StorageFileWritePosix *this);
+const String *storageFileWritePosixPath(StorageFileWritePosix *this);
+bool storageFileWritePosixSyncFile(StorageFileWritePosix *this);
+bool storageFileWritePosixSyncPath(StorageFileWritePosix *this);
+
+/***********************************************************************************************************************************
+Destructor
+***********************************************************************************************************************************/
+void storageFileWritePosixFree(StorageFileWritePosix *this);
+
+#endif

src/storage/file.h

@@ -1,35 +0,0 @@
-/***********************************************************************************************************************************
-Storage File
-***********************************************************************************************************************************/
-#ifndef STORAGE_FILE_H
-#define STORAGE_FILE_H
-
-/***********************************************************************************************************************************
-Storage file object
-***********************************************************************************************************************************/
-typedef struct StorageFile StorageFile;
-
-/***********************************************************************************************************************************
-Types of storage files, i.e. read or write.  The storage module does not allow files to be opened for both read and write since this
-is generally not supported by object stores.
-***********************************************************************************************************************************/
-typedef enum
-{
-    storageFileTypeRead,
-    storageFileTypeWrite,
-} StorageFileType;
-
-#include "storage/storage.h"
-
-/***********************************************************************************************************************************
-Functions
-***********************************************************************************************************************************/
-StorageFile *storageFileNew(const Storage *storage, String *name, StorageFileType type, void *data);
-
-void *storageFileData(const StorageFile *this);
-const String *storageFileName(const StorageFile *this);
-const Storage *storageFileStorage(const StorageFile *this);
-
-void storageFileFree(const StorageFile *this);
-
-#endif

src/storage/fileRead.c

@@ -0,0 +1,126 @@
+/***********************************************************************************************************************************
+Storage File Read
+***********************************************************************************************************************************/
+#include "common/assert.h"
+#include "common/memContext.h"
+#include "storage/fileRead.h"
+
+/***********************************************************************************************************************************
+Storage file structure
+***********************************************************************************************************************************/
+struct StorageFileRead
+{
+    MemContext *memContext;
+    StorageFileReadPosix *fileDriver;
+};
+
+/***********************************************************************************************************************************
+Create a new storage file
+***********************************************************************************************************************************/
+StorageFileRead *
+storageFileReadNew(const String *name, bool ignoreMissing, size_t bufferSize)
+{
+    StorageFileRead *this = NULL;
+
+    ASSERT_DEBUG(name != NULL);
+
+    MEM_CONTEXT_NEW_BEGIN("StorageFileRead")
+    {
+        this = memNew(sizeof(StorageFileRead));
+        this->memContext = memContextCurrent();
+
+        // Call driver function
+        this->fileDriver = storageFileReadPosixNew(name, ignoreMissing, bufferSize);
+    }
+    MEM_CONTEXT_NEW_END();
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Open the file
+***********************************************************************************************************************************/
+bool
+storageFileReadOpen(StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileReadPosixOpen(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Read data from the file
+***********************************************************************************************************************************/
+Buffer *
+storageFileRead(StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileReadPosix(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Move the file object to a new context
+***********************************************************************************************************************************/
+StorageFileRead *
+storageFileReadMove(StorageFileRead *this, MemContext *parentNew)
+{
+    if (this != NULL)
+        memContextMove(this->memContext, parentNew);
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Close the file
+***********************************************************************************************************************************/
+void
+storageFileReadClose(StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    storageFileReadPosixClose(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get file driver
+***********************************************************************************************************************************/
+StorageFileReadPosix *
+storageFileReadFileDriver(const StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->fileDriver;
+}
+
+/***********************************************************************************************************************************
+Should a missing file be ignored?
+***********************************************************************************************************************************/
+bool
+storageFileReadIgnoreMissing(const StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileReadPosixIgnoreMissing(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get file name
+***********************************************************************************************************************************/
+const String *
+storageFileReadName(const StorageFileRead *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileReadPosixName(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Free the file
+***********************************************************************************************************************************/
+void
+storageFileReadFree(StorageFileRead *this)
+{
+    if (this != NULL)
+        memContextFree(this->memContext);
+}

src/storage/fileRead.h

@@ -0,0 +1,42 @@
+/***********************************************************************************************************************************
+Storage File Read
+***********************************************************************************************************************************/
+#ifndef STORAGE_FILEREAD_H
+#define STORAGE_FILEREAD_H
+
+/***********************************************************************************************************************************
+Storage file read object
+***********************************************************************************************************************************/
+typedef struct StorageFileRead StorageFileRead;
+
+#include "common/type/buffer.h"
+#include "common/type/string.h"
+#include "storage/driver/posix/driverRead.h"
+
+/***********************************************************************************************************************************
+Constructor
+***********************************************************************************************************************************/
+StorageFileRead *storageFileReadNew(const String *name, bool ignoreMissing, size_t bufferSize);
+
+/***********************************************************************************************************************************
+Functions
+***********************************************************************************************************************************/
+bool storageFileReadOpen(StorageFileRead *this);
+Buffer *storageFileRead(StorageFileRead *this);
+void storageFileReadClose(StorageFileRead *this);
+
+StorageFileRead *storageFileReadMove(StorageFileRead *this, MemContext *parentNew);
+
+/***********************************************************************************************************************************
+Getters
+***********************************************************************************************************************************/
+StorageFileReadPosix *storageFileReadFileDriver(const StorageFileRead *this);
+bool storageFileReadIgnoreMissing(const StorageFileRead *this);
+const String *storageFileReadName(const StorageFileRead *this);
+
+/***********************************************************************************************************************************
+Destructor
+***********************************************************************************************************************************/
+void storageFileReadFree(StorageFileRead *this);
+
+#endif

src/storage/fileWrite.c

@@ -0,0 +1,202 @@
+/***********************************************************************************************************************************
+Storage File Write
+***********************************************************************************************************************************/
+#include "common/assert.h"
+#include "common/memContext.h"
+#include "storage/fileWrite.h"
+
+/***********************************************************************************************************************************
+Storage file structure
+***********************************************************************************************************************************/
+struct StorageFileWrite
+{
+    MemContext *memContext;
+    StorageFileWritePosix *fileDriver;
+};
+
+/***********************************************************************************************************************************
+Create a new storage file
+
+This object expects its context to be created in advance.  This is so the calling function can add whatever data it wants without
+required multiple functions and contexts to make it safe.
+***********************************************************************************************************************************/
+StorageFileWrite *
+storageFileWriteNew(
+    const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic)
+{
+    StorageFileWrite *this = NULL;
+
+    ASSERT_DEBUG(name != NULL);
+
+    // Create the file.  The file driver is not created here because we don't want to open the file for write until we know that
+    // there is a source file.
+    MEM_CONTEXT_NEW_BEGIN("StorageFileWrite")
+    {
+        this = memNew(sizeof(StorageFileWrite));
+        this->memContext = MEM_CONTEXT_NEW();
+
+        this->fileDriver = storageFileWritePosixNew(name, modeFile, modePath, noCreatePath, noSyncFile, noSyncPath, noAtomic);
+    }
+    MEM_CONTEXT_NEW_END();
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Open the file
+***********************************************************************************************************************************/
+void
+storageFileWriteOpen(StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    // Open the file
+    storageFileWritePosixOpen(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Write to a file
+***********************************************************************************************************************************/
+void
+storageFileWrite(StorageFileWrite *this, const Buffer *buffer)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    // Only write if there is data to write
+    if (buffer != NULL && bufSize(buffer) > 0)
+        storageFileWritePosix(this->fileDriver, buffer);
+}
+
+/***********************************************************************************************************************************
+Move the file object to a new context
+***********************************************************************************************************************************/
+StorageFileWrite *
+storageFileWriteMove(StorageFileWrite *this, MemContext *parentNew)
+{
+    if (this != NULL)
+        memContextMove(this->memContext, parentNew);
+
+    return this;
+}
+
+/***********************************************************************************************************************************
+Close the file
+***********************************************************************************************************************************/
+void
+storageFileWriteClose(StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    storageFileWritePosixClose(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Will the file be written atomically?
+
+Atomic writes means the file will be complete or be missing.  Filesystems have different ways to accomplish this.
+***********************************************************************************************************************************/
+bool
+storageFileWriteAtomic(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixAtomic(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Will the path be created if required?
+***********************************************************************************************************************************/
+bool
+storageFileWriteCreatePath(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixCreatePath(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get file driver
+***********************************************************************************************************************************/
+StorageFileWritePosix *
+storageFileWriteFileDriver(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return this->fileDriver;
+}
+
+/***********************************************************************************************************************************
+Get file mode
+***********************************************************************************************************************************/
+mode_t
+storageFileWriteModeFile(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixModeFile(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get path mode
+***********************************************************************************************************************************/
+mode_t
+storageFileWriteModePath(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixModePath(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get file name
+***********************************************************************************************************************************/
+const String *
+storageFileWriteName(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixName(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Get file path
+***********************************************************************************************************************************/
+const String *
+storageFileWritePath(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixPath(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Will the file be synced after it is closed?
+***********************************************************************************************************************************/
+bool
+storageFileWriteSyncFile(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixSyncFile(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Will the path be synced after the file is closed?
+***********************************************************************************************************************************/
+bool
+storageFileWriteSyncPath(const StorageFileWrite *this)
+{
+    ASSERT_DEBUG(this != NULL);
+
+    return storageFileWritePosixSyncPath(this->fileDriver);
+}
+
+/***********************************************************************************************************************************
+Free the file
+***********************************************************************************************************************************/
+void
+storageFileWriteFree(const StorageFileWrite *this)
+{
+    if (this != NULL)
+        memContextFree(this->memContext);
+}

src/storage/fileWrite.h

@@ -1,83 +1,57 @@
 /***********************************************************************************************************************************
-Storage File
+Storage File Write
 ***********************************************************************************************************************************/
-#include "common/assert.h"
+#ifndef STORAGE_FILEWRITE_H
-#include "common/memContext.h"
+#define STORAGE_FILEWRITE_H
-#include "storage/file.h"
+
+#include <sys/types.h>
 
 /***********************************************************************************************************************************
-Storage file structure
+Storage file read object
 ***********************************************************************************************************************************/
-struct StorageFile
+typedef struct StorageFileWrite StorageFileWrite;
-{
+
-    MemContext *memContext;
+#include "common/type/buffer.h"
-    const Storage *storage;
+#include "common/type/string.h"
-    String *name;
+#include "storage/driver/posix/driverWrite.h"
-    StorageFileType type;
+#include "version.h"
-    void *data;
-};
 
 /***********************************************************************************************************************************
-Create a new storage file
+Temporary file extension
-
-This object expects its context to be created in advance.  This is so the calling function can add whatever data it wants without
-required multiple functions and contexts to make it safe.
 ***********************************************************************************************************************************/
-StorageFile *storageFileNew(const Storage *storage, String *name, StorageFileType type, void *data)
+#define STORAGE_FILE_TEMP_EXT                                       PGBACKREST_BIN ".tmp"
-{
-    ASSERT_DEBUG(storage != NULL);
-    ASSERT_DEBUG(name != NULL);
-    ASSERT_DEBUG(data != NULL);
-
-    StorageFile *this = memNew(sizeof(StorageFile));
-    this->memContext = memContextCurrent();
-    this->storage = storage;
-    this->name = name;
-    this->type = type;
-    this->data = data;
-
-    return this;
-}
 
 /***********************************************************************************************************************************
-Get file data
+Constructor
 ***********************************************************************************************************************************/
-void *
+StorageFileWrite *storageFileWriteNew(
-storageFileData(const StorageFile *this)
+    const String *name, mode_t modeFile, mode_t modePath, bool noCreatePath, bool noSyncFile, bool noSyncPath, bool noAtomic);
-{
-    ASSERT_DEBUG(this != NULL);
-
-    return this->data;
-}
 
 /***********************************************************************************************************************************
-Get file name
+Functions
 ***********************************************************************************************************************************/
-const String *
+void storageFileWriteOpen(StorageFileWrite *this);
-storageFileName(const StorageFile *this)
+void storageFileWrite(StorageFileWrite *this, const Buffer *buffer);
-{
+void storageFileWriteClose(StorageFileWrite *this);
-    ASSERT_DEBUG(this != NULL);
 
-    return this->name;
+StorageFileWrite *storageFileWriteMove(StorageFileWrite *this, MemContext *parentNew);
-}
 
 /***********************************************************************************************************************************
-Get file storage object
+Getters
 ***********************************************************************************************************************************/
-const Storage *
+bool storageFileWriteAtomic(const StorageFileWrite *this);
-storageFileStorage(const StorageFile *this)
+bool storageFileWriteCreatePath(const StorageFileWrite *this);
-{
+StorageFileWritePosix *storageFileWriteFileDriver(const StorageFileWrite *this);
-    ASSERT_DEBUG(this != NULL);
+mode_t storageFileWriteModeFile(const StorageFileWrite *this);
-
+mode_t storageFileWriteModePath(const StorageFileWrite *this);
-    return this->storage;
+const String *storageFileWriteName(const StorageFileWrite *this);
-}
+const String *storageFileWritePath(const StorageFileWrite *this);
+bool storageFileWriteSyncFile(const StorageFileWrite *this);
+bool storageFileWriteSyncPath(const StorageFileWrite *this);
 
 /***********************************************************************************************************************************
-Free the file
+Destructor
 ***********************************************************************************************************************************/
-void
+void storageFileWriteFree(const StorageFileWrite *this);
-storageFileFree(const StorageFile *this)
+
-{
+#endif
-    if (this != NULL)
-        memContextFree(this->memContext);
-}

src/storage/storage.c

@@ -6,7 +6,7 @@ Storage Manager
 #include "common/assert.h"
 #include "common/memContext.h"
 #include "common/wait.h"
-#include "storage/driver/posix.h"
+#include "storage/driver/posix/driver.h"
 #include "storage/storage.h"
 
 /***********************************************************************************************************************************
@@ -60,12 +60,42 @@ storageNew(const String *path, StorageNewParam param)
 }
 
 /***********************************************************************************************************************************
-Get storage buffer size
+Copy a file
 ***********************************************************************************************************************************/
-size_t
+bool
-storageBufferSize(const Storage *this)
+storageCopy(StorageFileRead *source, StorageFileWrite *destination)
 {
-    return this->bufferSize;
+    bool result = false;
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Open source file
+        if (storageFileReadOpen(source))
+        {
+            // Open the destination file now that we know the source file exists and is readable
+            storageFileWriteOpen(destination);
+
+            // Copy data from source to destination
+            Buffer *read = NULL;
+
+            do
+            {
+                read = storageFileRead(source);
+                storageFileWrite(destination, read);
+            }
+            while (read != NULL);
+
+            // Close the source and destination files
+            storageFileReadClose(source);
+            storageFileWriteClose(destination);
+
+            // Set result to indicate that the file was copied
+            result = true;
+        }
+    }
+    MEM_CONTEXT_TEMP_END();
+
+    return result;
 }
 
 /***********************************************************************************************************************************
@@ -104,13 +134,37 @@ storageExists(const Storage *this, const String *pathExp, StorageExistsParam par
 Read from storage into a buffer
 ***********************************************************************************************************************************/
 Buffer *
-storageGet(const StorageFile *file)
+storageGet(StorageFileRead *file)
 {
     Buffer *result = NULL;
 
-    // Call driver function if a file was passed
+    ASSERT_DEBUG(file != NULL);
-    if (file != NULL)
+
-        result = storageDriverPosixGet(file);
+    // If the file exists
+    if (storageFileReadOpen(file))
+    {
+        MEM_CONTEXT_TEMP_BEGIN()
+        {
+            result = bufNew(0);
+            Buffer *read = NULL;
+
+            do
+            {
+                // Read data
+                read = storageFileRead(file);
+
+                // Add to result and free read buffer
+                bufCat(result, read);
+                bufFree(read);
+            }
+            while (read != NULL);
+
+            bufMove(result, MEM_CONTEXT_OLD());
+        }
+        MEM_CONTEXT_TEMP_END();
+
+        storageFileReadClose(file);
+    }
 
     return result;
 }
@@ -136,25 +190,48 @@ storageList(const Storage *this, const String *pathExp, StorageListParam param)
     return result;
 }
 
+/***********************************************************************************************************************************
+Move a file
+***********************************************************************************************************************************/
+void
+storageMove(StorageFileRead *source, StorageFileWrite *destination)
+{
+    ASSERT_DEBUG(!storageFileReadIgnoreMissing(source));
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // If the file can't be moved it will need to be copied
+        if (!storageDriverPosixMove(storageFileReadFileDriver(source), storageFileWriteFileDriver(destination)))
+        {
+            // Perform the copy
+            storageCopyNP(source, destination);
+
+            // Remove the source file
+            storageDriverPosixRemove(storageFileReadName(source), false);
+
+            // Sync source path if the destination path was synced.  We know the source and destination paths are different because
+            // the move did not succeed.  This will need updating when drivers other than Posix/CIFS are implemented.
+            if (storageFileWriteSyncPath(destination))
+                storageDriverPosixPathSync(strPath(storageFileReadName(source)), false);
+        }
+    }
+    MEM_CONTEXT_TEMP_END();
+}
 
 /***********************************************************************************************************************************
 Open a file for reading
 ***********************************************************************************************************************************/
-StorageFile *
+StorageFileRead *
-storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam param)
+storageNewRead(const Storage *this, const String *fileExp, StorageNewReadParam param)
 {
-    StorageFile *result = NULL;
+    StorageFileRead *result = NULL;
 
-    MEM_CONTEXT_NEW_BEGIN("StorageFile")
+    MEM_CONTEXT_NEW_BEGIN("StorageFileRead")
     {
         String *fileName = storagePathNP(this, fileExp);
 
-        // Call driver function
+        // Create the file
-        void *data = storageDriverPosixOpenRead(fileName, param.ignoreMissing);
+        result = storageFileReadMove(storageFileReadNew(fileName, param.ignoreMissing, this->bufferSize), MEM_CONTEXT_OLD());
-
-        // Free mem contexts if missing files are ignored
-        if (data != NULL)
-            result = storageFileNew(this, fileName, storageFileTypeRead, data);
     }
     MEM_CONTEXT_NEW_END();
 
@@ -164,23 +241,26 @@ storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam
 /***********************************************************************************************************************************
 Open a file for writing
 ***********************************************************************************************************************************/
-StorageFile *
+StorageFileWrite *
-storageOpenWrite(const Storage *this, const String *fileExp, StorageOpenWriteParam param)
+storageNewWrite(const Storage *this, const String *fileExp, StorageNewWriteParam param)
 {
+    StorageFileWrite *result = NULL;
+
     ASSERT_STORAGE_ALLOWS_WRITE();
 
-    StorageFile *result = NULL;
+    MEM_CONTEXT_TEMP_BEGIN()
-
-    MEM_CONTEXT_NEW_BEGIN("StorageFile")
     {
         String *fileName = storagePathNP(this, fileExp);
 
-        // Call driver function
+        // Create the file
-        result = storageFileNew(
+        result = storageFileWriteMove(
-            this, fileName, storageFileTypeWrite,
+            storageFileWriteNew(
-            storageDriverPosixOpenWrite(fileName, param.mode != 0 ? param.mode : this->modeFile));
+                fileName, param.modeFile != 0 ? param.modeFile : this->modeFile,
+                param.modePath != 0 ? param.modePath : this->modePath, param.noCreatePath, param.noSyncFile, param.noSyncPath,
+                param.noAtomic),
+            MEM_CONTEXT_OLD());
     }
-    MEM_CONTEXT_NEW_END();
+    MEM_CONTEXT_TEMP_END();
 
     return result;
 }
@@ -323,15 +403,33 @@ storagePathRemove(const Storage *this, const String *pathExp, StoragePathRemoveP
     MEM_CONTEXT_TEMP_END();
 }
 
+/***********************************************************************************************************************************
+Sync a path
+***********************************************************************************************************************************/
+void storagePathSync(const Storage *this, const String *pathExp, StoragePathSyncParam param)
+{
+    ASSERT_STORAGE_ALLOWS_WRITE();
+
+    MEM_CONTEXT_TEMP_BEGIN()
+    {
+        // Build the path
+        String *path = storagePathNP(this, pathExp);
+
+        // Call driver function
+        storageDriverPosixPathSync(path, param.ignoreMissing);
+    }
+    MEM_CONTEXT_TEMP_END();
+}
+
 /***********************************************************************************************************************************
 Write a buffer to storage
 ***********************************************************************************************************************************/
 void
-storagePut(const StorageFile *file, const Buffer *buffer)
+storagePut(StorageFileWrite *file, const Buffer *buffer)
 {
-    // Write data if buffer is not null.  Otherwise, an empty file is expected.
+    storageFileWriteOpen(file);
-    if (buffer != NULL)
+    storageFileWrite(file, buffer);
-        storageDriverPosixPut(file, buffer);
+    storageFileWriteClose(file);
 }
 
 /***********************************************************************************************************************************

src/storage/storage.h

@@ -13,7 +13,8 @@ typedef struct Storage Storage;
 
 #include "common/type/buffer.h"
 #include "common/type/stringList.h"
-#include "storage/file.h"
+#include "storage/fileRead.h"
+#include "storage/fileWrite.h"
 
 /***********************************************************************************************************************************
 Default buffer size
@@ -29,7 +30,6 @@ Default file and path modes
 #define STORAGE_FILE_MODE_DEFAULT                                   0640
 #define STORAGE_PATH_MODE_DEFAULT                                   0750
 
-
 /***********************************************************************************************************************************
 Path expression callback function type - used to modify paths based on expressions enclosed in <>
 ***********************************************************************************************************************************/
@@ -54,6 +54,14 @@ typedef struct StorageNewParam
 
 Storage *storageNew(const String *path, StorageNewParam param);
 
+/***********************************************************************************************************************************
+storageCopy
+***********************************************************************************************************************************/
+#define storageCopyNP(source, destination)                                                                                         \
+    storageCopy(source, destination)
+
+bool storageCopy(StorageFileRead *source, StorageFileWrite *destination);
+
 /***********************************************************************************************************************************
 storageExists
 ***********************************************************************************************************************************/
@@ -75,7 +83,7 @@ storageGet
 #define storageGetNP(file)                                                                                                         \
     storageGet(file)
 
-Buffer *storageGet(const StorageFile *file);
+Buffer *storageGet(StorageFileRead *file);
 
 /***********************************************************************************************************************************
 storageList
@@ -94,34 +102,47 @@ typedef struct StorageListParam
 StringList *storageList(const Storage *this, const String *pathExp, StorageListParam param);
 
 /***********************************************************************************************************************************
-storageOpenRead
+storageMove
 ***********************************************************************************************************************************/
-typedef struct StorageOpenReadParam
+#define storageMoveNP(source, destination)                                                                                         \
-{
+    storageMove(source, destination)
-    bool ignoreMissing;
-} StorageOpenReadParam;
 
-#define storageOpenReadP(this, pathExp, ...)                                                                                       \
+void storageMove(StorageFileRead *source, StorageFileWrite *destination);
-    storageOpenRead(this, pathExp, (StorageOpenReadParam){__VA_ARGS__})
-#define storageOpenReadNP(this, pathExp)                                                                                           \
-    storageOpenRead(this, pathExp, (StorageOpenReadParam){0})
-
-StorageFile *storageOpenRead(const Storage *this, const String *fileExp, StorageOpenReadParam param);
 
 /***********************************************************************************************************************************
-storageOpenWrite
+storageNewRead
 ***********************************************************************************************************************************/
-typedef struct StorageOpenWriteParam
+typedef struct StorageNewReadParam
 {
-    mode_t mode;
+    bool ignoreMissing;
-} StorageOpenWriteParam;
+} StorageNewReadParam;
 
-#define storageOpenWriteP(this, pathExp, ...)                                                                                      \
+#define storageNewReadP(this, pathExp, ...)                                                                                        \
-    storageOpenWrite(this, pathExp, (StorageOpenWriteParam){__VA_ARGS__})
+    storageNewRead(this, pathExp, (StorageNewReadParam){__VA_ARGS__})
-#define storageOpenWriteNP(this, pathExp)                                                                                          \
+#define storageNewReadNP(this, pathExp)                                                                                            \
-    storageOpenWrite(this, pathExp, (StorageOpenWriteParam){0})
+    storageNewRead(this, pathExp, (StorageNewReadParam){0})
 
-StorageFile *storageOpenWrite(const Storage *this, const String *fileExp, StorageOpenWriteParam param);
+StorageFileRead *storageNewRead(const Storage *this, const String *fileExp, StorageNewReadParam param);
+
+/***********************************************************************************************************************************
+storageNewWrite
+***********************************************************************************************************************************/
+typedef struct StorageNewWriteParam
+{
+    mode_t modeFile;
+    mode_t modePath;
+    bool noCreatePath;
+    bool noSyncFile;
+    bool noSyncPath;
+    bool noAtomic;
+} StorageNewWriteParam;
+
+#define storageNewWriteP(this, pathExp, ...)                                                                                       \
+    storageNewWrite(this, pathExp, (StorageNewWriteParam){__VA_ARGS__})
+#define storageNewWriteNP(this, pathExp)                                                                                           \
+    storageNewWrite(this, pathExp, (StorageNewWriteParam){0})
+
+StorageFileWrite *storageNewWrite(const Storage *this, const String *fileExp, StorageNewWriteParam param);
 
 /***********************************************************************************************************************************
 storagePath
@@ -164,13 +185,28 @@ typedef struct StoragePathRemoveParam
 
 void storagePathRemove(const Storage *this, const String *pathExp, StoragePathRemoveParam param);
 
+/***********************************************************************************************************************************
+storagePathSync
+***********************************************************************************************************************************/
+typedef struct StoragePathSync
+{
+    bool ignoreMissing;
+} StoragePathSyncParam;
+
+#define storagePathSyncP(this, pathExp, ...)                                                                                       \
+    storagePathSync(this, pathExp, (StoragePathSyncParam){__VA_ARGS__})
+#define storagePathSyncNP(this, pathExp)                                                                                           \
+    storagePathSync(this, pathExp, (StoragePathSyncParam){0})
+
+void storagePathSync(const Storage *this, const String *pathExp, StoragePathSyncParam param);
+
 /***********************************************************************************************************************************
 storagePut
 ***********************************************************************************************************************************/
 #define storagePutNP(file, buffer)                                                                                                 \
     storagePut(file, buffer)
 
-void storagePut(const StorageFile *file, const Buffer *buffer);
+void storagePut(StorageFileWrite *file, const Buffer *buffer);
 
 /***********************************************************************************************************************************
 storageRemove
@@ -195,9 +231,4 @@ storageFree
 
 void storageFree(const Storage *this);
 
-/***********************************************************************************************************************************
-Functions
-***********************************************************************************************************************************/
-size_t storageBufferSize(const Storage *this);
-
 #endif

test/define.yaml

@@ -466,19 +466,23 @@ module:
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: file
-        total: 1
+        total: 3
         c: true
 
         coverage:
-          storage/file: full
+          storage/driver/posix/driverFile: full
+          storage/driver/posix/driverRead: full
+          storage/driver/posix/driverWrite: full
+          storage/fileRead: full
+          storage/fileWrite: full
 
       # ----------------------------------------------------------------------------------------------------------------------------
       - name: storage
-        total: 10
+        total: 13
         c: true
 
         coverage:
-          storage/driver/posix: full
+          storage/driver/posix/driver: full
           storage/storage: full
 
       # ----------------------------------------------------------------------------------------------------------------------------

test/src/common/logTest.c

@@ -50,7 +50,7 @@ After the comparison the log is cleared so the next result can be compared.
 void
 testLogResult(const char *expected)
 {
-    String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stdoutFile))));
+    String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stdoutFile))));
 
     if (!strEqZ(actual, expected))
         THROW(AssertError, "\n\nexpected log:\n\n%s\n\nbut actual log was:\n\n%s\n\n", expected, strPtr(actual));
@@ -67,7 +67,7 @@ After the comparison the log is cleared so the next result can be compared.
 void
 testLogErrResult(const char *expected)
 {
-    String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stderrFile))));
+    String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stderrFile))));
 
     if (!strEqZ(actual, expected))
         THROW(AssertError, "\n\nexpected error log:\n\n%s\n\nbut actual error log was:\n\n%s\n\n", expected, strPtr(actual));
@@ -82,12 +82,12 @@ Make sure nothing is left in the log after all tests have completed
 void
 testLogFinal()
 {
-    String *actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stdoutFile))));
+    String *actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stdoutFile))));
 
     if (!strEqZ(actual, ""))
         THROW(AssertError, "\n\nexpected log to be empty but actual log was:\n\n%s\n\n", strPtr(actual));
 
-    actual = strTrim(strNewBuf(storageGetNP(storageOpenReadNP(storageLocal(), stderrFile))));
+    actual = strTrim(strNewBuf(storageGetNP(storageNewReadNP(storageLocal(), stderrFile))));
 
     if (!strEqZ(actual, ""))
         THROW(AssertError, "\n\nexpected error log to be empty but actual error log was:\n\n%s\n\n", strPtr(actual));

test/src/module/archive/pushTest.c

@@ -40,28 +40,28 @@ testRun()
 
         // -------------------------------------------------------------------------------------------------------------------------
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
             bufNewStr(strNew(BOGUS_STR)));
         TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok content must have at least two lines");
 
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
             bufNewStr(strNew(BOGUS_STR "\n")));
         TEST_ERROR(walStatus(segment, false), FormatError, "000000010000000100000001.ok message must be > 0");
 
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
             bufNewStr(strNew(BOGUS_STR "\nmessage")));
         TEST_ERROR(walStatus(segment, false), FormatError, "unable to convert str 'BOGUS' to int");
 
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
             bufNewStr(strNew("0\nwarning")));
         TEST_RESULT_BOOL(walStatus(segment, false), true, "ok file with warning");
         testLogResult("P00   WARN: warning");
 
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.ok", strPtr(segment))),
             bufNewStr(strNew("25\nerror")));
         TEST_RESULT_BOOL(walStatus(segment, false), true, "error status renamed to ok");
         testLogResult(
@@ -69,7 +69,7 @@ testRun()
 
         // -------------------------------------------------------------------------------------------------------------------------
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
             bufNewStr(strNew("")));
         TEST_ERROR(
             walStatus(segment, false), AssertError,
@@ -81,7 +81,7 @@ testRun()
         TEST_ERROR(walStatus(segment, true), AssertError, "status file '000000010000000100000001.error' has no content");
 
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
+            storageNewWriteNP(storageSpool(), strNewFmt(STORAGE_SPOOL_ARCHIVE_OUT "/%s.error", strPtr(segment))),
             bufNewStr(strNew("25\nmessage")));
         TEST_ERROR(walStatus(segment, true), AssertError, "message");
 
@@ -138,7 +138,7 @@ testRun()
         mkdir(strPtr(strNewFmt("%s/archive", testPath())), 0750);
         mkdir(strPtr(strNewFmt("%s/archive/db", testPath())), 0750);
         mkdir(strPtr(strNewFmt("%s/archive/db/out", testPath())), 0750);
-        storagePutNP(storageOpenWriteNP(storageSpool(), errorFile), bufNewStr(strNew("25\n" BOGUS_STR)));
+        storagePutNP(storageNewWriteNP(storageSpool(), errorFile), bufNewStr(strNew("25\n" BOGUS_STR)));
 
         TEST_ERROR(cmdArchivePush(), AssertError, BOGUS_STR);
 
@@ -147,7 +147,7 @@ testRun()
         // Write out a valid ok file and test for success
         // -------------------------------------------------------------------------------------------------------------------------
         storagePutNP(
-            storageOpenWriteNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")),
+            storageNewWriteNP(storageSpool(), strNew(STORAGE_SPOOL_ARCHIVE_OUT "/000000010000000100000001.ok")),
             bufNewStr(strNew("")));
 
         TEST_RESULT_VOID(cmdArchivePush(), "successful push");

test/src/module/common/iniTest.c

@@ -88,7 +88,7 @@ testRun()
             "\n"
         );
 
-        TEST_RESULT_VOID(storagePutNP(storageOpenWriteNP(storageLocalWrite(), fileName), bufNewStr(content)), "put ini to file");
+        TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageLocalWrite(), fileName), bufNewStr(content)), "put ini to file");
         TEST_RESULT_VOID(iniLoad(ini, fileName), "load ini from file");
 
         TEST_RESULT_STR(strPtr(varStr(iniGet(ini, strNew("global"), strNew("compress")))), "y", "get compress");

test/src/module/common/logTest.c

@@ -128,7 +128,7 @@ testRun()
         Storage *storage = storageNewNP(strNew(testPath()));
 
         TEST_RESULT_STR(
-            strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stdoutFile)))),
+            strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stdoutFile)))),
             "P00   WARN: format 99\n"
             "P00  ERROR: [026]: message\n"
             "P00  ERROR: [026]: message1\n"
@@ -137,7 +137,7 @@ testRun()
 
         // Check stderr
         TEST_RESULT_STR(
-            strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stderrFile)))),
+            strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stderrFile)))),
             "DEBUG: test.c:test_func(): message\n"
             "INFO: info message\n"
             "WARN: unable to open log file '/BOGUS': Permission denied\n"
@@ -146,7 +146,7 @@ testRun()
 
         // Check file
         TEST_RESULT_STR(
-            strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, fileFile)))),
+            strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, fileFile)))),
             "-------------------PROCESS START-------------------\n"
             "P00  DEBUG: test.c:test_func(): message\n"
             "\n"

test/src/module/config/parseTest.c

@@ -55,14 +55,14 @@ testRun()
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
         storagePut(
-            storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(
+            storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(
                 strNew(
                     "[global]\n"
                     "compress-level=3\n"
                     "spool-path=/path/to/spool\n")));
 
         storagePut(
-            storageOpenWriteNP(
+            storageNewWriteNP(
                     storageLocalWrite(), strNewFmt("%s/global-backup.conf", strPtr(configIncludePath))), bufNewStr(
                     strNew(
                         "[global:backup]\n"
@@ -77,14 +77,14 @@ testRun()
                         "buffer-size=65536\n")));
 
         storagePut(
-            storageOpenWriteNP(storageLocalWrite(), strNewFmt("%s/db-backup.conf", strPtr(configIncludePath))), bufNewStr(
+            storageNewWriteNP(storageLocalWrite(), strNewFmt("%s/db-backup.conf", strPtr(configIncludePath))), bufNewStr(
                 strNew(
                     "[db:backup]\n"
                     "compress=n\n"
                     "recovery-option=a=b\n")));
 
         storagePut(
-            storageOpenWriteNP(storageLocalWrite(), strNewFmt("%s/stanza.db.conf", strPtr(configIncludePath))), bufNewStr(
+            storageNewWriteNP(storageLocalWrite(), strNewFmt("%s/stanza.db.conf", strPtr(configIncludePath))), bufNewStr(
                 strNew(
                     "[db]\n"
                     "pg1-host=db\n"
@@ -215,7 +215,7 @@ testRun()
 
         mkdir(strPtr(strPath(oldConfigDefault)), 0750);
         storagePut(
-            storageOpenWriteNP(
+            storageNewWriteNP(
                     storageLocalWrite(), oldConfigDefault), bufNewStr(
                     strNew(
                         "[global:backup]\n"
@@ -747,7 +747,7 @@ testRun()
         strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global]\n"
             "compress=bogus\n"
         )));
@@ -762,7 +762,7 @@ testRun()
         strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global]\n"
             "compress=\n"
         )));
@@ -778,7 +778,7 @@ testRun()
         strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[db]\n"
             "pg1-path=/path/to/db\n"
             "db-path=/also/path/to/db\n"
@@ -795,7 +795,7 @@ testRun()
         strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[db]\n"
             "pg1-path=/path/to/db\n"
             "pg1-path=/also/path/to/db\n"
@@ -897,7 +897,7 @@ testRun()
         strLstAdd(argList, strNew("--reset-backup-standby"));
         strLstAdd(argList, strNew(TEST_COMMAND_BACKUP));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global]\n"
             "compress-level=3\n"
             "spool-path=/path/to/spool\n"
@@ -958,7 +958,7 @@ testRun()
         strLstAdd(argList, strNew("--archive-push-queue-max=4503599627370496"));
         strLstAdd(argList, strNew("archive-push"));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global]\n"
             "spool-path=/path/to/spool\n"
         )));
@@ -1029,7 +1029,7 @@ testRun()
         strLstAdd(argList, strNew("--stanza=db"));
         strLstAdd(argList, strNew(TEST_COMMAND_RESTORE));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global:restore]\n"
             "recovery-option=f=g\n"
             "recovery-option=hijk=l\n"
@@ -1051,7 +1051,7 @@ testRun()
         strLstAdd(argList, strNewFmt("--config=%s", strPtr(configFile)));
         strLstAdd(argList, strNew("info"));
 
-        storagePutNP(storageOpenWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
+        storagePutNP(storageNewWriteNP(storageLocalWrite(), configFile), bufNewStr(strNew(
             "[global]\n"
             "repo1-path=/path/to/repo\n"
             "\n"

test/src/module/help/helpTest.c

@@ -305,6 +305,6 @@ testRun()
         dup2(stdoutSave, STDOUT_FILENO);
 
         Storage *storage = storageNewNP(strNew(testPath()));
-        TEST_RESULT_STR(strPtr(strNewBuf(storageGetNP(storageOpenReadNP(storage, stdoutFile)))), generalHelp, "    check text");
+        TEST_RESULT_STR(strPtr(strNewBuf(storageGetNP(storageNewReadNP(storage, stdoutFile)))), generalHelp, "    check text");
     }
 }

test/src/module/storage/fileTest.c

@@ -1,6 +1,22 @@
 /***********************************************************************************************************************************
 Test Storage File
 ***********************************************************************************************************************************/
+#include "storage/storage.h"
+
+/***********************************************************************************************************************************
+Get the mode of a file on local storage
+***********************************************************************************************************************************/
+mode_t
+storageStatMode(const String *path)
+{
+    // Attempt to stat the file
+    struct stat statFile;
+
+    if (stat(strPtr(path), &statFile) == -1)                                                // {uncovered - error should not happen}
+        THROW_SYS_ERROR(FileOpenError, "unable to stat '%s'", strPtr(path));                // {uncovered+}
+
+    return statFile.st_mode & 0777;
+}
 
 /***********************************************************************************************************************************
 Test Run
@@ -8,26 +24,266 @@ Test Run
 void
 testRun()
 {
+    // Create default storage object for testing
+    Storage *storageTest = storageNewP(strNew(testPath()), .write = true, .bufferSize = 2);
+
+    // Create a directory and file that cannot be accessed to test permissions errors
+    String *fileNoPerm = strNewFmt("%s/noperm/noperm", testPath());
+    String *pathNoPerm = strPath(fileNoPerm);
+
+    TEST_RESULT_INT(
+        system(
+            strPtr(strNewFmt("sudo mkdir -m 700 %s && sudo touch %s && sudo chmod 600 %s", strPtr(pathNoPerm), strPtr(fileNoPerm),
+            strPtr(fileNoPerm)))),
+        0, "create no perm path/file");
+
     // *****************************************************************************************************************************
-    if (testBegin("storageFileNew() and storageFileFree()"))
+    if (testBegin("StorageFile"))
     {
-        MemContext *memContext = NULL;
+        TEST_ERROR_FMT(
-        StorageFile *file = NULL;
+            storageFilePosixOpen(pathNoPerm, O_RDONLY, 0, false, &PathOpenError, "test"), PathOpenError,
+            "unable to open '%s' for test: [13] Permission denied", strPtr(pathNoPerm));
 
-        MEM_CONTEXT_NEW_BEGIN("TestFile")
+        // -------------------------------------------------------------------------------------------------------------------------
+        String *fileName = strNewFmt("%s/test.file", testPath());
+
+        TEST_ERROR_FMT(
+            storageFilePosixOpen(fileName, O_RDONLY, 0, false, &FileOpenError, "read"), FileOpenError,
+            "unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
+
+        TEST_RESULT_INT(storageFilePosixOpen(fileName, O_RDONLY, 0, true, &FileOpenError, "read"), -1, "missing file ignored");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        int handle = -1;
+
+        TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileName)))), 0, "create read file");
+        TEST_ASSIGN(handle, storageFilePosixOpen(fileName, O_RDONLY, 0, false, &FileOpenError, "read"), "open read file");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ERROR_FMT(
+            storageFilePosixSync(-99, fileName, &PathSyncError, false), PathSyncError,
+            "unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
+        TEST_ERROR_FMT(
+            storageFilePosixSync(-99, fileName, &FileSyncError, true), FileSyncError,
+            "unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
+
+        TEST_RESULT_VOID(storageFilePosixSync(handle, fileName, &FileSyncError, false), "sync file");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ERROR_FMT(
+            storageFilePosixClose(-99, fileName, &FileCloseError), FileCloseError,
+            "unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
+
+        TEST_RESULT_VOID(storageFilePosixClose(handle, fileName, &FileSyncError), "close file");
+
+        TEST_RESULT_INT(system(strPtr(strNewFmt("rm %s", strPtr(fileName)))), 0, "remove read file");
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("StorageFileRead"))
+    {
+        StorageFileRead *file = NULL;
+
+        TEST_ASSIGN(file, storageNewReadP(storageTest, fileNoPerm, .ignoreMissing = true), "new read file");
+        TEST_RESULT_PTR(storageFileReadFileDriver(file), file->fileDriver, "    check file driver");
+        TEST_RESULT_BOOL(storageFileReadIgnoreMissing(file), true, "    check ignore missing");
+        TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileNoPerm), "    check name");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ASSIGN(file, storageNewReadNP(storageTest, fileNoPerm), "new no perm read file");
+        TEST_ERROR_FMT(
+            storageFileReadOpen(file), FileOpenError,
+            "unable to open '%s' for read: [13] Permission denied", strPtr(fileNoPerm));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        String *fileName = strNewFmt("%s/test.file", testPath());
+
+        TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new missing read file");
+        TEST_ERROR_FMT(
+            storageFileReadOpen(file), FileOpenError,
+            "unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ASSIGN(file, storageNewReadP(storageTest, fileName, .ignoreMissing = true), "new missing read file");
+        TEST_RESULT_BOOL(storageFileReadOpen(file), false, "   missing file ignored");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        Buffer *expectedBuffer = bufNewStr(strNew("TESTFILE\n"));
+        TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, fileName), expectedBuffer), "write test file");
+
+        TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new read file");
+        TEST_RESULT_BOOL(storageFileReadOpen(file), true, "   open file");
+
+        // Close the file handle so operations will fail
+        close(file->fileDriver->handle);
+
+        TEST_ERROR_FMT(
+            storageFileRead(file), FileReadError,
+            "unable to read '%s': [9] Bad file descriptor", strPtr(fileName));
+        TEST_ERROR_FMT(
+            storageFileReadClose(file), FileCloseError,
+            "unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
+
+        // Set file handle to -1 so the close on free with not fail
+        file->fileDriver->handle = -1;
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        Buffer *buffer = bufNew(0);
+
+        MEM_CONTEXT_TEMP_BEGIN()
         {
-            memContext = MEM_CONTEXT_NEW();
+            TEST_ASSIGN(file, storageFileReadMove(storageNewReadNP(storageTest, fileName), MEM_CONTEXT_OLD()), "new read file");
-            TEST_ASSIGN(file, storageFileNew((Storage *)1, strNew("file"), storageFileTypeRead, (void *)2), "new file");
         }
-        MEM_CONTEXT_NEW_END();
+        MEM_CONTEXT_TEMP_END();
 
-        TEST_RESULT_PTR(storageFileData(file), (void *)2, "    check data");
+        TEST_RESULT_BOOL(storageFileReadOpen(file), true, "   open file");
-        TEST_RESULT_PTR(file->memContext, memContext, "    check mem context");
+        TEST_RESULT_STR(strPtr(storageFileReadName(file)), strPtr(fileName), "    check file name");
-        TEST_RESULT_STR(strPtr(storageFileName(file)), "file", "    check name");
-        TEST_RESULT_INT(file->type, storageFileTypeRead, "    check type");
-        TEST_RESULT_PTR(storageFileStorage(file), (Storage *)1, "    check storage");
 
-        TEST_RESULT_VOID(storageFileFree(file), "free file");
+        TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), "    load data");
-        TEST_RESULT_VOID(storageFileFree(NULL), "free null file");
+        TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), "    load data");
+        TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), "    load data");
+        TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), "    load data");
+        TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), false, "    check file contents (not all loaded yet)");
+
+        TEST_RESULT_VOID(bufCat(buffer, storageFileRead(file)), "    load data");
+        TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, "    check file contents (all loaded)");
+
+        TEST_RESULT_PTR(storageFileRead(file), NULL, "    eof");
+        TEST_RESULT_PTR(storageFileRead(file), NULL, "    still eof");
+
+        TEST_RESULT_VOID(storageFileReadClose(file), "    close file");
+        TEST_RESULT_VOID(storageFileReadClose(file), "    close again");
+
+        TEST_RESULT_VOID(storageFileReadFree(file), "   free file");
+        TEST_RESULT_VOID(storageFileReadFree(NULL), "   free null file");
+        TEST_RESULT_VOID(storageFileReadPosixFree(NULL), "   free null file");
+
+        TEST_RESULT_VOID(storageFileReadMove(NULL, NULL), "   move null file");
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("StorageFileWrite"))
+    {
+        StorageFileWrite *file = NULL;
+
+        TEST_ASSIGN(
+            file,
+            storageNewWriteP(
+                storageTest, fileNoPerm, .modeFile = 0444, .modePath = 0555, .noCreatePath = true, .noSyncFile = true,
+                .noSyncPath = true, .noAtomic = true),
+            "new write file");
+
+        TEST_RESULT_BOOL(storageFileWriteAtomic(file), false, "    check atomic");
+        TEST_RESULT_BOOL(storageFileWriteCreatePath(file), false, "    check create path");
+        TEST_RESULT_PTR(storageFileWriteFileDriver(file), file->fileDriver, "    check file driver");
+        TEST_RESULT_INT(storageFileWriteModeFile(file), 0444, "    check mode file");
+        TEST_RESULT_INT(storageFileWriteModePath(file), 0555, "    check mode path");
+        TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileNoPerm), "    check name");
+        TEST_RESULT_STR(strPtr(storageFileWritePath(file)), strPtr(strPath(fileNoPerm)), "    check path");
+        TEST_RESULT_BOOL(storageFileWriteSyncPath(file), false, "    check sync path");
+        TEST_RESULT_BOOL(storageFileWriteSyncFile(file), false, "    check sync file");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ASSIGN(file, storageNewWriteP(storageTest, fileNoPerm, .noAtomic = true), "new write file");
+        TEST_ERROR_FMT(
+            storageFileWriteOpen(file), FileOpenError,
+            "unable to open '%s' for write: [13] Permission denied", strPtr(fileNoPerm));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        String *fileName = strNewFmt("%s/sub1/test.file", testPath());
+
+        TEST_ASSIGN(file, storageNewWriteP(storageTest, fileName, .noCreatePath = true, .noAtomic = true), "new write file");
+        TEST_ERROR_FMT(
+            storageFileWriteOpen(file), FileOpenError,
+            "unable to open '%s' for write: [2] No such file or directory", strPtr(fileName));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        String *fileTmp = strNewFmt("%s.pgbackrest.tmp", strPtr(fileName));
+        Buffer *buffer = bufNewStr(strNew("TESTFILE\n"));
+
+        TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file");
+        TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileName), "    check file name");
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+
+        // Close the file handle so operations will fail
+        close(file->fileDriver->handle);
+        storageRemoveP(storageTest, fileTmp, .errorOnMissing = true);
+
+        TEST_ERROR_FMT(
+            storageFileWrite(file, buffer), FileWriteError,
+            "unable to write '%s': [9] Bad file descriptor", strPtr(fileName));
+        TEST_ERROR_FMT(
+            storageFileWriteClose(file), FileSyncError,
+            "unable to sync '%s': [9] Bad file descriptor", strPtr(fileName));
+
+        // Disable file sync so the close can be reached
+        file->fileDriver->noSyncFile = true;
+
+        TEST_ERROR_FMT(
+            storageFileWriteClose(file), FileCloseError,
+            "unable to close '%s': [9] Bad file descriptor", strPtr(fileName));
+
+        // Set file handle to -1 so the close on free with not fail
+        file->fileDriver->handle = -1;
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file");
+        TEST_RESULT_STR(strPtr(storageFileWriteName(file)), strPtr(fileName), "    check file name");
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+
+        // Rename the file back to original name from tmp -- this will cause the rename in close to fail
+        TEST_RESULT_INT(rename(strPtr(fileTmp), strPtr(fileName)), 0, " rename tmp file");
+        TEST_ERROR_FMT(
+            storageFileWriteClose(file), FileMoveError,
+            "unable to move '%s' to '%s': [2] No such file or directory", strPtr(fileTmp), strPtr(fileName));
+
+        // Set file handle to -1 so the close on free with not fail
+        file->fileDriver->handle = -1;
+
+        storageRemoveP(storageTest, fileName, .errorOnMissing = true);
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        MEM_CONTEXT_TEMP_BEGIN()
+        {
+            TEST_ASSIGN(file, storageFileWriteMove(storageNewWriteNP(storageTest, fileName), MEM_CONTEXT_OLD()), "new write file");
+        }
+        MEM_CONTEXT_TEMP_END();
+
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+        TEST_RESULT_VOID(storageFileWrite(file, NULL), "   write null buffer to file");
+        TEST_RESULT_VOID(storageFileWrite(file, bufNew(0)), "   write zero buffer to file");
+        TEST_RESULT_VOID(storageFileWrite(file, buffer), "   write to file");
+        TEST_RESULT_VOID(storageFileWriteClose(file), "   close file");
+        TEST_RESULT_VOID(storageFileWriteFree(file), "   free file");
+        TEST_RESULT_VOID(storageFileWriteFree(NULL), "   free null file");
+        TEST_RESULT_VOID(storageFileWritePosixFree(NULL), "   free null posix file");
+        TEST_RESULT_VOID(storageFileWriteMove(NULL, NULL), "   move null file");
+
+        Buffer *expectedBuffer = storageGetNP(storageNewReadNP(storageTest, fileName));
+        TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, "    check file contents");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0750, "    check path mode");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, "    check file mode");
+
+        storageRemoveP(storageTest, fileName, .errorOnMissing = true);
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        fileName = strNewFmt("%s/sub2/test.file", testPath());
+
+        TEST_ASSIGN(
+            file,
+            storageNewWriteP(
+                storageTest, fileName, .modePath = 0700, .modeFile = 0600, .noSyncPath = true, .noSyncFile = true,
+                .noAtomic = true),
+            "new write file");
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+        TEST_RESULT_VOID(storageFileWrite(file, buffer), "   write to file");
+        TEST_RESULT_VOID(storageFileWriteClose(file), "   close file");
+
+        expectedBuffer = storageGetNP(storageNewReadNP(storageTest, fileName));
+        TEST_RESULT_BOOL(bufEq(buffer, expectedBuffer), true, "    check file contents");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0700, "    check path mode");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0600, "    check file mode");
+
+        storageRemoveP(storageTest, fileName, .errorOnMissing = true);
     }
 }

test/src/module/storage/helperTest.c

@@ -22,7 +22,7 @@ testRun()
 
         TEST_RESULT_STR(strPtr(storagePathNP(storage, NULL)), "/", "check base path");
 
-        TEST_ERROR(storageOpenWriteNP(storage, writeFile), AssertError, "assertion 'this->write == true' failed");
+        TEST_ERROR(storageNewWriteNP(storage, writeFile), AssertError, "assertion 'this->write == true' failed");
     }
     // -----------------------------------------------------------------------------------------------------------------------------
     if (testBegin("storageLocalWrite()"))
@@ -36,7 +36,7 @@ testRun()
 
         TEST_RESULT_STR(strPtr(storagePathNP(storage, NULL)), "/", "check base path");
 
-        TEST_RESULT_VOID(storageOpenWriteNP(storage, writeFile), "writes are allowed");
+        TEST_RESULT_VOID(storageNewWriteNP(storage, writeFile), "writes are allowed");
     }
 
     // -----------------------------------------------------------------------------------------------------------------------------
@@ -65,6 +65,6 @@ testRun()
 
         TEST_ERROR(storagePathNP(storage, strNew("<" BOGUS_STR ">")), AssertError, "invalid expression '<BOGUS>'");
 
-        TEST_RESULT_VOID(storageOpenWriteNP(storage, writeFile), "writes are allowed");
+        TEST_RESULT_VOID(storageNewWriteNP(storage, writeFile), "writes are allowed");
     }
 }

test/src/module/storage/storageTest.c

@@ -2,7 +2,8 @@
 Test Storage Manager
 ***********************************************************************************************************************************/
 #include "common/time.h"
-#include "storage/file.h"
+#include "storage/fileRead.h"
+#include "storage/fileWrite.h"
 
 /***********************************************************************************************************************************
 Get the mode of a file on local storage
@@ -42,7 +43,8 @@ void
 testRun()
 {
     // Create default storage object for testing
-    Storage *storageTest = storageNewP(strNew(testPath()), .write = true);
+    Storage *storageTest = storageNewP(strNew(testPath()), .write = true, .bufferSize = 3);
+    Storage *storageTmp = storageNewP(strNew("/tmp"), .write = true);
 
     // Create a directory and file that cannot be accessed to test permissions errors
     String *fileNoPerm = strNewFmt("%s/noperm/noperm", testPath());
@@ -52,7 +54,7 @@ testRun()
         system(
             strPtr(strNewFmt("sudo mkdir -m 700 %s && sudo touch %s && sudo chmod 600 %s", strPtr(pathNoPerm), strPtr(fileNoPerm),
             strPtr(fileNoPerm)))),
-        0, "create no perm file");
+        0, "create no perm path/file");
 
     // *****************************************************************************************************************************
     if (testBegin("storageNew() and storageFree()"))
@@ -94,9 +96,9 @@ testRun()
         TEST_RESULT_BOOL(storageExistsP(storageTest, strNew("missing"), .timeout = .1), false, "file does not exist");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageExistsNP(storageTest, fileNoPerm), FileOpenError,
-            strPtr(strNewFmt("unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm))));
+            "unable to stat '%s': [13] Permission denied", strPtr(fileNoPerm));
 
         // -------------------------------------------------------------------------------------------------------------------------
         String *fileExists = strNewFmt("%s/exists", testPath());
@@ -121,35 +123,149 @@ testRun()
     if (testBegin("storageList()"))
     {
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageListP(storageTest, strNew(BOGUS_STR), .errorOnMissing = true), PathOpenError,
-            strPtr(strNewFmt("unable to open path '%s/BOGUS' for read: [2] No such file or directory", testPath())));
+            "unable to open path '%s/BOGUS' for read: [2] No such file or directory", testPath());
 
         TEST_RESULT_PTR(storageListNP(storageTest, strNew(BOGUS_STR)), NULL, "ignore missing dir");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageListNP(storageTest, pathNoPerm), PathOpenError,
-            strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm))));
+            "unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm));
 
         // Should still error even when ignore missing
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageListNP(storageTest, pathNoPerm), PathOpenError,
-            strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm))));
+            "unable to open path '%s' for read: [13] Permission denied", strPtr(pathNoPerm));
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_VOID(
-            storagePutNP(storageOpenWriteNP(storageTest, strNew("aaa.txt")), bufNewStr(strNew("aaa"))), "write aaa.text");
+            storagePutNP(storageNewWriteNP(storageTest, strNew("aaa.txt")), bufNewStr(strNew("aaa"))), "write aaa.text");
         TEST_RESULT_STR(
             strPtr(strLstJoin(storageListNP(storageTest, NULL), ", ")), "aaa.txt, stderr.log, stdout.log, noperm",
             "dir list");
 
         TEST_RESULT_VOID(
-            storagePutNP(storageOpenWriteNP(storageTest, strNew("bbb.txt")), bufNewStr(strNew("bbb"))), "write bbb.text");
+            storagePutNP(storageNewWriteNP(storageTest, strNew("bbb.txt")), bufNewStr(strNew("bbb"))), "write bbb.text");
         TEST_RESULT_STR(
             strPtr(strLstJoin(storageListP(storageTest, NULL, .expression = strNew("^bbb")), ", ")), "bbb.txt", "dir list");
     }
 
+    // *****************************************************************************************************************************
+    if (testBegin("storageCopy()"))
+    {
+        String *sourceFile = strNewFmt("%s/source.txt", testPath());
+        String *destinationFile = strNewFmt("%s/destination.txt", testPath());
+
+        StorageFileRead *source = storageNewReadNP(storageTest, sourceFile);
+        StorageFileWrite *destination = storageNewWriteNP(storageTest, destinationFile);
+
+        TEST_ERROR_FMT(
+            storageCopyNP(source, destination), FileOpenError,
+            "unable to open '%s' for read: [2] No such file or directory", strPtr(sourceFile));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        source = storageNewReadP(storageTest, sourceFile, .ignoreMissing = true);
+
+        TEST_RESULT_BOOL(storageCopyNP(source, destination), false, "copy and ignore missing file");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        Buffer *expectedBuffer = bufNewStr(strNew("TESTFILE\n"));
+        TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, sourceFile), expectedBuffer), "write source file");
+
+        source = storageNewReadNP(storageTest, sourceFile);
+
+        TEST_RESULT_BOOL(storageCopyNP(source, destination), true, "copy file");
+        TEST_RESULT_BOOL(bufEq(expectedBuffer, storageGetNP(storageNewReadNP(storageTest, destinationFile))), true, "check file");
+
+        storageRemoveP(storageTest, sourceFile, .errorOnMissing = true);
+        storageRemoveP(storageTest, destinationFile, .errorOnMissing = true);
+    }
+
+    // *****************************************************************************************************************************
+    if (testBegin("storageMove()"))
+    {
+        String *sourceFile = strNewFmt("%s/source.txt", testPath());
+        String *destinationFile = strNewFmt("%s/sub/destination.txt", testPath());
+
+        StorageFileRead *source = storageNewReadNP(storageTest, sourceFile);
+        StorageFileWrite *destination = storageNewWriteNP(storageTest, destinationFile);
+
+        TEST_ERROR_FMT(
+            storageMoveNP(source, destination), FileMissingError,
+            "unable to move missing file '%s': [2] No such file or directory", strPtr(sourceFile));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        source = storageNewReadNP(storageTest, fileNoPerm);
+
+        TEST_ERROR_FMT(
+            storageMoveNP(source, destination), FileMoveError,
+            "unable to move '%s' to '%s': [13] Permission denied", strPtr(fileNoPerm), strPtr(destinationFile));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        Buffer *buffer = bufNewStr(strNew("TESTFILE"));
+        storagePutNP(storageNewWriteNP(storageTest, sourceFile), buffer);
+
+        source = storageNewReadNP(storageTest, sourceFile);
+        destination = storageNewWriteP(storageTest, destinationFile, .noCreatePath = true);
+
+        TEST_ERROR_FMT(
+            storageMoveNP(source, destination), PathMissingError,
+            "unable to move '%s' to missing path '%s': [2] No such file or directory", strPtr(sourceFile),
+            strPtr(strPath(destinationFile)));
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        destination = storageNewWriteNP(storageTest, destinationFile);
+
+        TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to subpath");
+        TEST_RESULT_BOOL(storageExistsNP(storageTest, sourceFile), false, "check source file not exists");
+        TEST_RESULT_BOOL(storageExistsNP(storageTest, destinationFile), true, "check destination file exists");
+        TEST_RESULT_STR(
+            strPtr(strNewBuf(storageGetNP(storageNewReadNP(storageTest, destinationFile)))), "TESTFILE",
+            "check destination contents");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        sourceFile = destinationFile;
+        source = storageNewReadNP(storageTest, sourceFile);
+        destinationFile = strNewFmt("%s/sub/destination2.txt", testPath());
+        destination = storageNewWriteNP(storageTest, destinationFile);
+
+        TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to same path");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        sourceFile = destinationFile;
+        source = storageNewReadNP(storageTest, sourceFile);
+        destinationFile = strNewFmt("%s/source.txt", testPath());
+        destination = storageNewWriteP(storageTest, destinationFile, .noSyncPath = true);
+
+        TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to parent path (no sync)");
+
+        // Move across filesystems
+        // -------------------------------------------------------------------------------------------------------------------------
+        sourceFile = destinationFile;
+        source = storageNewReadNP(storageTest, sourceFile);
+        destinationFile = strNewFmt("/tmp/destination.txt");
+        destination = storageNewWriteNP(storageTmp, destinationFile);
+
+        TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to another filesystem");
+        TEST_RESULT_BOOL(storageExistsNP(storageTest, sourceFile), false, "check source file not exists");
+        TEST_RESULT_BOOL(storageExistsNP(storageTmp, destinationFile), true, "check destination file exists");
+
+        // Move across fileystems without syncing the paths
+        // -------------------------------------------------------------------------------------------------------------------------
+        sourceFile = destinationFile;
+        source = storageNewReadNP(storageTmp, sourceFile);
+        destinationFile = strNewFmt("%s/source.txt", testPath());
+        destination = storageNewWriteP(storageTest, destinationFile, .noSyncPath = true);
+
+        TEST_RESULT_VOID(storageMoveNP(source, destination), "move file to another filesystem without path sync");
+        TEST_RESULT_BOOL(storageExistsNP(storageTmp, sourceFile), false, "check source file not exists");
+        TEST_RESULT_BOOL(storageExistsNP(storageTest, destinationFile), true, "check destination file exists");
+
+        storageRemoveP(storageTest, destinationFile, .errorOnMissing = true);
+    }
+
     // *****************************************************************************************************************************
     if (testBegin("storagePath()"))
     {
@@ -200,16 +316,16 @@ testRun()
         TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub1")), "create sub1");
         TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strNew("sub1"))), 0750, "check sub1 dir mode");
         TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub1")), "create sub1 again");
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathCreateP(storageTest, strNew("sub1"), .errorOnExists = true), PathCreateError,
-            strPtr(strNewFmt("unable to create path '%s/sub1': [17] File exists", testPath())));
+            "unable to create path '%s/sub1': [17] File exists", testPath());
 
         TEST_RESULT_VOID(storagePathCreateP(storageTest, strNew("sub2"), .mode = 0777), "create sub2 with custom mode");
         TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strNew("sub2"))), 0777, "check sub2 dir mode");
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathCreateP(storageTest, strNew("sub3/sub4"), .noParentCreate = true), PathCreateError,
-            strPtr(strNewFmt("unable to create path '%s/sub3/sub4': [2] No such file or directory", testPath())));
+            "unable to create path '%s/sub3/sub4': [2] No such file or directory", testPath());
         TEST_RESULT_VOID(storagePathCreateNP(storageTest, strNew("sub3/sub4")), "create sub3/sub4");
 
         TEST_RESULT_INT(system(strPtr(strNewFmt("rm -rf %s/sub*", testPath()))), 0, "remove sub paths");
@@ -221,9 +337,9 @@ testRun()
         // -------------------------------------------------------------------------------------------------------------------------
         String *pathRemove1 = strNewFmt("%s/remove1", testPath());
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathRemoveP(storageTest, pathRemove1, .errorOnMissing = true), PathRemoveError,
-            strPtr(strNewFmt("unable to remove path '%s': [2] No such file or directory", strPtr(pathRemove1))));
+            "unable to remove path '%s': [2] No such file or directory", strPtr(pathRemove1));
         TEST_RESULT_VOID(storagePathRemoveP(storageTest, pathRemove1, .recurse = true), "ignore missing path");
 
         // -------------------------------------------------------------------------------------------------------------------------
@@ -231,19 +347,19 @@ testRun()
 
         TEST_RESULT_INT(system(strPtr(strNewFmt("sudo mkdir -p -m 700 %s", strPtr(pathRemove2)))), 0, "create noperm paths");
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathRemoveNP(storageTest, pathRemove2), PathRemoveError,
-            strPtr(strNewFmt("unable to remove path '%s': [13] Permission denied", strPtr(pathRemove2))));
+            "unable to remove path '%s': [13] Permission denied", strPtr(pathRemove2));
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
-            strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2))));
+            "unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2));
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove1)))), 0, "top path can be removed");
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathRemoveP(storageTest, pathRemove2, .recurse = true), PathOpenError,
-            strPtr(strNewFmt("unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2))));
+            "unable to open path '%s' for read: [13] Permission denied", strPtr(pathRemove2));
 
         // -------------------------------------------------------------------------------------------------------------------------
         String *fileRemove = strNewFmt("%s/remove.txt", strPtr(pathRemove2));
@@ -254,9 +370,9 @@ testRun()
                 strPtr(fileRemove)))),
             0, "add no perm file");
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storagePathRemoveP(storageTest, pathRemove1, .recurse = true), PathRemoveError,
-            strPtr(strNewFmt("unable to remove path/file '%s': [13] Permission denied", strPtr(fileRemove))));
+            "unable to remove path/file '%s': [13] Permission denied", strPtr(fileRemove));
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_INT(system(strPtr(strNewFmt("sudo chmod 777 %s", strPtr(pathRemove2)))), 0, "bottom path can be removed");
@@ -276,46 +392,72 @@ testRun()
     }
 
     // *****************************************************************************************************************************
-    if (testBegin("storageOpenRead()"))
+    if (testBegin("storagePathSync()"))
     {
-        TEST_ERROR(
+        TEST_ERROR_FMT(
-            storageOpenReadNP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR)),
+            storagePathSyncNP(storageTest, fileNoPerm), PathOpenError,
-            FileOpenError,
+            "unable to open '%s' for sync: [13] Permission denied", strPtr(fileNoPerm));
-            strPtr(strNewFmt("unable to open '%s/%s' for read: [2] No such file or directory", testPath(), BOGUS_STR)));
-
-        TEST_RESULT_PTR(
-            storageOpenReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true),
-            NULL, "open missing file without error");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ERROR(
+        String *pathName = strNewFmt("%s/testpath", testPath());
-            storageGetNP(storageOpenReadP(storageTest, fileNoPerm, .ignoreMissing = true)), FileOpenError,
+
-            strPtr(strNewFmt("unable to open '%s' for read: [13] Permission denied", strPtr(fileNoPerm))));
+        TEST_ERROR_FMT(
+            storagePathSyncNP(storageTest, pathName), PathOpenError,
+            "unable to open '%s' for sync: [2] No such file or directory", strPtr(pathName));
+
+        TEST_RESULT_VOID(storagePathSyncP(storageTest, pathName, .ignoreMissing = true), "ignore missing path");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        TEST_RESULT_VOID(storagePathCreateNP(storageTest, pathName), "create path to sync");
+        TEST_RESULT_VOID(storagePathSyncNP(storageTest, pathName), "sync path");
     }
 
     // *****************************************************************************************************************************
-    if (testBegin("storageOpenWrite()"))
+    if (testBegin("storageNewRead()"))
     {
-        TEST_ERROR(
+        StorageFileRead *file = NULL;
-            storageOpenWriteNP(storageTest, strNew(testPath())), FileOpenError,
+        String *fileName = strNewFmt("%s/readtest.txt", testPath());
-            strPtr(strNewFmt("unable to open '%s' for write: [21] Is a directory", testPath())));
+
+        TEST_ASSIGN(file, storageNewReadNP(storageTest, fileName), "new read file (defaults)");
+        TEST_ERROR_FMT(
+            storageFileReadOpen(file), FileOpenError,
+            "unable to open '%s' for read: [2] No such file or directory", strPtr(fileName));
 
         // -------------------------------------------------------------------------------------------------------------------------
-        StorageFile *file = NULL;
+        TEST_RESULT_INT(system(strPtr(strNewFmt("touch %s", strPtr(fileName)))), 0, "create read file");
-        String *fileName = strNewFmt("%s/testfile", testPath());
 
-        TEST_ASSIGN(file, storageOpenWriteNP(storageTest, fileName), "open file for write (defaults)");
+        TEST_RESULT_BOOL(storageFileReadOpen(file), true, "    open file");
-        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, "check file mode");
+        TEST_RESULT_VOID(storageFileReadClose(file), "    close file");
-        close(STORAGE_DATA(file)->handle);
+    }
 
-        storageRemoveP(storageTest, fileName, .errorOnMissing = true);
+    // *****************************************************************************************************************************
+    if (testBegin("storageNewWrite()"))
+    {
+        StorageFileWrite *file = NULL;
+
+        TEST_ASSIGN(file, storageNewWriteP(storageTest, fileNoPerm, .noAtomic = true), "new write file (defaults)");
+        TEST_ERROR_FMT(
+            storageFileWriteOpen(file), FileOpenError,
+            "unable to open '%s' for write: [13] Permission denied", strPtr(fileNoPerm));
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ASSIGN(file, storageOpenWriteP(storageTest, fileName, .mode = 0777), "open file for write (custom)");
+        String *fileName = strNewFmt("%s/sub1/testfile", testPath());
-        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0777, "check file mode");
-        close(STORAGE_DATA(file)->handle);
 
-        storageRemoveP(storageTest, fileName, .errorOnMissing = true);
+        TEST_ASSIGN(file, storageNewWriteNP(storageTest, fileName), "new write file (defaults)");
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+        TEST_RESULT_VOID(storageFileWriteClose(file), "   close file");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0750, "    check path mode");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0640, "    check file mode");
+
+        // -------------------------------------------------------------------------------------------------------------------------
+        fileName = strNewFmt("%s/sub2/testfile", testPath());
+
+        TEST_ASSIGN(
+            file, storageNewWriteP(storageTest, fileName, .modePath = 0700, .modeFile = 0600), "new write file (set mode)");
+        TEST_RESULT_VOID(storageFileWriteOpen(file), "    open file");
+        TEST_RESULT_VOID(storageFileWriteClose(file), "   close file");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, strPath(fileName))), 0700, "    check path mode");
+        TEST_RESULT_INT(storageStatMode(storagePath(storageTest, fileName)), 0600, "    check file mode");
     }
 
     // *****************************************************************************************************************************
@@ -323,42 +465,32 @@ testRun()
     {
         Storage *storageTest = storageNewP(strNew("/"), .write = true);
 
-        TEST_ERROR(
+        TEST_ERROR_FMT(
-            storageGetNP(storageOpenReadNP(storageTest, strNew(testPath()))), FileReadError,
+            storageGetNP(storageNewReadNP(storageTest, strNew(testPath()))), FileReadError,
-            strPtr(strNewFmt("unable to read '%s': [21] Is a directory", testPath())));
+            "unable to read '%s': [21] Is a directory", testPath());
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_RESULT_VOID(
+        String *emptyFile = strNewFmt("%s/test.empty", testPath());
-            storagePutNP(storageOpenWriteNP(storageTest, strNewFmt("%s/test.empty", testPath())), NULL), "put empty file");
+        TEST_RESULT_VOID(storagePutNP(storageNewWriteNP(storageTest, emptyFile), NULL), "put empty file");
+        TEST_RESULT_BOOL(storageExistsNP(storageTest, emptyFile), true, "check empty file exists");
 
         // -------------------------------------------------------------------------------------------------------------------------
         Buffer *buffer = bufNewStr(strNew("TESTFILE\n"));
-        StorageFile *file = NULL;
 
-        StorageFileDataPosix fileData = {.handle = 999999};
-        MEM_CONTEXT_NEW_BEGIN("FileTest")
-        {
-            file = storageFileNew(storageTest, strNew("badfile"), storageFileTypeWrite, &fileData);
-        }
-        MEM_CONTEXT_NEW_END();
-
-        TEST_ERROR(storagePutNP(file, buffer), FileWriteError, "unable to write 'badfile': [9] Bad file descriptor");
-
-        // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_VOID(
-            storagePutNP(storageOpenWriteNP(storageTest, strNewFmt("%s/test.txt", testPath())), buffer), "put text file");
+            storagePutNP(storageNewWriteNP(storageTest, strNewFmt("%s/test.txt", testPath())), buffer), "put text file");
 
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_PTR(
-            storageGetNP(storageOpenReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true)),
+            storageGetNP(storageNewReadP(storageTest, strNewFmt("%s/%s", testPath(), BOGUS_STR), .ignoreMissing = true)),
             NULL, "get missing file");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.empty", testPath()))), "get empty");
+        TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.empty", testPath()))), "get empty");
         TEST_RESULT_INT(bufSize(buffer), 0, "size is 0");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
+        TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
         TEST_RESULT_INT(bufSize(buffer), 9, "check size");
         TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TESTFILE\n", bufSize(buffer)) == 0, true, "check content");
 
@@ -366,7 +498,7 @@ testRun()
         const Storage *storage = storageTest;
         ((Storage *)storage)->bufferSize = 2;
 
-        TEST_ASSIGN(buffer, storageGetNP(storageOpenReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
+        TEST_ASSIGN(buffer, storageGetNP(storageNewReadNP(storageTest, strNewFmt("%s/test.txt", testPath()))), "get text");
         TEST_RESULT_INT(bufSize(buffer), 9, "check size");
         TEST_RESULT_BOOL(memcmp(bufPtr(buffer), "TESTFILE\n", bufSize(buffer)) == 0, true, "check content");
     }
@@ -376,9 +508,9 @@ testRun()
     {
         // -------------------------------------------------------------------------------------------------------------------------
         TEST_RESULT_VOID(storageRemoveNP(storageTest, strNew("missing")), "remove missing file");
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageRemoveP(storageTest, strNew("missing"), .errorOnMissing = true), FileRemoveError,
-            strPtr(strNewFmt("unable to remove '%s/missing': [2] No such file or directory", testPath())));
+            "unable to remove '%s/missing': [2] No such file or directory", testPath());
 
         // -------------------------------------------------------------------------------------------------------------------------
         String *fileExists = strNewFmt("%s/exists", testPath());
@@ -387,8 +519,8 @@ testRun()
         TEST_RESULT_VOID(storageRemoveNP(storageTest, fileExists), "remove exists file");
 
         // -------------------------------------------------------------------------------------------------------------------------
-        TEST_ERROR(
+        TEST_ERROR_FMT(
             storageRemoveNP(storageTest, fileNoPerm), FileRemoveError,
-            strPtr(strNewFmt("unable to remove '%s': [13] Permission denied", strPtr(fileNoPerm))));
+            "unable to remove '%s': [13] Permission denied", strPtr(fileNoPerm));
     }
 }