diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 69eb5857c..5414f3787 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,6 +61,34 @@ jobs: - name: Run Test run: cd ${HOME?} && ${GITHUB_WORKSPACE?}/pgbackrest/test/ci.pl ${{matrix.param}} --param=build-max=2 + # C test harness. Run inside a container so tests that require socket binding work correctly. + testc: + runs-on: ubuntu-22.04 + container: + image: ubuntu:22.04 + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + with: + path: pgbackrest + + - name: Install + run: | + apt-get update + DEBCONF_NONINTERACTIVE_SEEN=true DEBIAN_FRONTEND=noninteractive apt-get install -y sudo zlib1g-dev libssl-dev libxml2-dev libpq-dev libyaml-dev pkg-config gcc ccache meson liblz4-dev liblz4-tool zstd libzstd-dev bzip2 libbz2-dev tzdata + adduser --disabled-password --gecos \"\" runner + + - name: Build + run: | + sudo -u runner cp -rp ${GITHUB_WORKSPACE?}/pgbackrest /home/runner + sudo -u runner meson -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug /home/runner/test/build/none /home/runner/pgbackrest + sudo -u runner ninja -C /home/runner/test/build/none test/src/test-pgbackrest + + - name: Test + run: | + sudo -u runner /home/runner/test/build/none/test/src/test-pgbackrest --repo-path=/home/runner/pgbackrest --test-path=/home/runner/test --no-repo-copy test + # Basic tests on other architectures using emulation. The emulation is so slow that running all the unit tests would be too # expensive, but this at least shows that the build works and some of the more complex tests run. In particular, it is good to # test on one big-endian architecture to be sure that checksums are correct. diff --git a/test/src/build/config/config.yaml b/test/src/build/config/config.yaml index 574c6fab6..1f35bef43 100644 --- a/test/src/build/config/config.yaml +++ b/test/src/build/config/config.yaml @@ -10,7 +10,8 @@ command: log-level-default: DEBUG parameter-allowed: true - test: {} + test: + parameter-allowed: true noop: internal: true @@ -44,16 +45,18 @@ option: - 512KiB - 1MiB - module: - type: string - command: - test: {} - neutral-umask: type: boolean internal: true default: true + repo-copy: + type: boolean + default: true + negate: true + command: + test: {} + repo-path: type: string default: pgbackrest diff --git a/test/src/build/help/help.xml b/test/src/build/help/help.xml index 0c75423fe..8c216df0e 100644 --- a/test/src/build/help/help.xml +++ b/test/src/build/help/help.xml @@ -77,16 +77,6 @@ n - - Module to test. - - - Unit test module to be tested. - - - common/error - - Use a neutral umask. @@ -99,6 +89,16 @@ n + + Make copy of code repository. + + + Make a copy of the code repository for testing so changes to the code repository do not affect tests. + + + n + + Path to code repository. @@ -106,7 +106,7 @@ Path to the original code repository or a copy. - n + /path/to/pgbackrest diff --git a/test/src/command/help/meson.build b/test/src/command/help/meson.build index 41dd2d87f..83cc41bc7 100644 --- a/test/src/command/help/meson.build +++ b/test/src/command/help/meson.build @@ -12,6 +12,6 @@ test_help_auto_c_inc = custom_target( build_code, 'help', '@CURRENT_SOURCE_DIR@/../../..', - '@BUILD_ROOT@/test/src', + '@BUILD_ROOT@/test', ], ) diff --git a/test/src/command/test/build.c b/test/src/command/test/build.c index ebdeaa9b9..64c23a961 100644 --- a/test/src/command/test/build.c +++ b/test/src/command/test/build.c @@ -30,7 +30,7 @@ Constants TestBuild * testBldNew( const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId, - const String *const moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime, + const TestDefModule *const module, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime, const String *const timeZone) { FUNCTION_LOG_BEGIN(logLevelDebug); @@ -38,7 +38,7 @@ testBldNew( FUNCTION_LOG_PARAM(STRING, pathTest); FUNCTION_LOG_PARAM(STRING, vm); FUNCTION_LOG_PARAM(UINT, vmId); - FUNCTION_LOG_PARAM(STRING, moduleName); + FUNCTION_LOG_PARAM_P(VOID, module); FUNCTION_LOG_PARAM(UINT, test); FUNCTION_LOG_PARAM(UINT64, scale); FUNCTION_LOG_PARAM(ENUM, logLevel); @@ -49,7 +49,7 @@ testBldNew( ASSERT(pathRepo != NULL); ASSERT(pathTest != NULL); ASSERT(vm != NULL); - ASSERT(moduleName != NULL); + ASSERT(module != NULL); ASSERT(scale != 0); TestBuild *this = NULL; @@ -67,7 +67,7 @@ testBldNew( .pathTest = strDup(pathTest), .vm = strDup(vm), .vmId = vmId, - .moduleName = strDup(moduleName), + .module = module, .test = test, .scale = scale, .logLevel = logLevel, @@ -78,10 +78,6 @@ testBldNew( this->pub.storageRepo = storagePosixNewP(testBldPathRepo(this)); this->pub.storageTest = storagePosixNewP(testBldPathTest(this)); - - // Find the module to test - this->pub.module = lstFind(testDefParse(testBldStorageRepo(this)).moduleList, &this->pub.moduleName); - CHECK(AssertError, this->pub.module != NULL, "unable to find module"); } OBJ_NEW_END(); @@ -451,8 +447,7 @@ testBldUnit(TestBuild *const this) // Path to the project exe when it exists const String *const pathProjectExe = storagePathP( - testBldStorageTest(this), - strNewFmt("bin/%s%s/" PROJECT_BIN, strZ(testBldVm(this)), strEqZ(testBldVm(this), "none") ? "/src" : "")); + testBldStorageTest(this), strNewFmt("build/%s/src/" PROJECT_BIN, strZ(testBldVm(this)))); strReplace(testC, STRDEF("{[C_TEST_PROJECT_EXE]}"), pathProjectExe); // Path to source -- used to construct __FILENAME__ tests @@ -490,7 +485,7 @@ testBldUnit(TestBuild *const this) // Include test file strReplace( testC, STRDEF("{[C_TEST_INCLUDE]}"), - strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepo), strZ(bldEnum(NULL, testBldModuleName(this))))); + strNewFmt("#include \"%s/test/src/module/%sTest.c\"", strZ(pathRepo), strZ(bldEnum(NULL, module->name)))); // Test list String *const testList = strNew(); diff --git a/test/src/command/test/build.h b/test/src/command/test/build.h index 490b3bd0b..2b6c31e2d 100644 --- a/test/src/command/test/build.h +++ b/test/src/command/test/build.h @@ -19,7 +19,7 @@ typedef struct TestBuild TestBuild; Constructors ***********************************************************************************************************************************/ TestBuild *testBldNew( - const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName, + const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const TestDefModule *module, unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone); /*********************************************************************************************************************************** @@ -33,7 +33,6 @@ typedef struct TestBuildPub const Storage *storageTest; // Test storage const String *vm; // Vm to run the test on unsigned int vmId; // Vm id (0-based) to run the test on - const String *moduleName; // Module to run const TestDefModule *module; // Module definition unsigned int test; // Specific test to run (0 if all) LogLevel logLevel; // Test log level @@ -93,13 +92,6 @@ testBldModule(const TestBuild *const this) return THIS_PUB(TestBuild)->module; } -// Module to run -__attribute__((always_inline)) static inline const String * -testBldModuleName(const TestBuild *const this) -{ - return THIS_PUB(TestBuild)->moduleName; -} - // Specific test to run __attribute__((always_inline)) static inline unsigned int testBldTest(const TestBuild *const this) diff --git a/test/src/command/test/test.c b/test/src/command/test/test.c index b4b45fc1e..9dc733d12 100644 --- a/test/src/command/test/test.c +++ b/test/src/command/test/test.c @@ -11,67 +11,235 @@ Test Command #include "config/config.h" #include "storage/posix/storage.h" +/**********************************************************************************************************************************/ +static const String *cmdTestExecLog; + +static void +cmdTestExec(const String *const command) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STRING, command); + FUNCTION_LOG_END(); + + ASSERT(cmdTestExecLog != NULL); + ASSERT(command != NULL); + + LOG_DETAIL_FMT("exec: %s", strZ(command)); + + if (system(zNewFmt("%s > %s 2>&1", strZ(command), strZ(cmdTestExecLog))) != 0) + { + const Buffer *const buffer = storageGetP( + storageNewReadP(storagePosixNewP(FSLASH_STR), cmdTestExecLog, .ignoreMissing = true)); + + THROW_FMT( + ExecuteError, "unable to execute: %s > %s 2>&1:%s", strZ(command), strZ(cmdTestExecLog), + buffer == NULL || bufEmpty(buffer) ? " no log output" : zNewFmt("\n%s", strZ(strNewBuf(buffer)))); + } + + FUNCTION_LOG_RETURN_VOID(); +} + +/**********************************************************************************************************************************/ +static void +cmdTestPathCreate(const Storage *const storage, const String *const path) +{ + FUNCTION_LOG_BEGIN(logLevelDebug); + FUNCTION_LOG_PARAM(STORAGE, storage); + FUNCTION_LOG_PARAM(STRING, path); + FUNCTION_LOG_END(); + + ASSERT(storage != NULL); + + TRY_BEGIN() + { + storagePathRemoveP(storage, path, .recurse = true); + } + CATCH_ANY() + { + // Reset permissions + cmdTestExec(strNewFmt("chmod -R 777 %s", strZ(storagePathP(storage, path)))); + + // Try to remove again + storagePathRemoveP(storage, path, .recurse = true); + } + TRY_END(); + + storagePathCreateP(storage, path, .mode = 0770); + + FUNCTION_LOG_RETURN_VOID(); +} + /**********************************************************************************************************************************/ void cmdTest( const String *const pathRepo, const String *const pathTest, const String *const vm, const unsigned int vmId, - const String *const moduleName, const unsigned int test, const uint64_t scale, const LogLevel logLevel, const bool logTime, - const String *const timeZone) + const StringList *moduleFilterList, const unsigned int test, const uint64_t scale, const LogLevel logLevel, + const bool logTime, const String *const timeZone, bool repoCopy) { FUNCTION_LOG_BEGIN(logLevelDebug); FUNCTION_LOG_PARAM(STRING, pathRepo); FUNCTION_LOG_PARAM(STRING, pathTest); FUNCTION_LOG_PARAM(STRING, vm); FUNCTION_LOG_PARAM(UINT, vmId); - FUNCTION_LOG_PARAM(STRING, moduleName); + FUNCTION_LOG_PARAM(STRING_LIST, moduleFilterList); FUNCTION_LOG_PARAM(UINT, test); FUNCTION_LOG_PARAM(UINT64, scale); FUNCTION_LOG_PARAM(ENUM, logLevel); FUNCTION_LOG_PARAM(BOOL, logTime); FUNCTION_LOG_PARAM(STRING, timeZone); + FUNCTION_LOG_PARAM(BOOL, repoCopy); FUNCTION_LOG_END(); MEM_CONTEXT_TEMP_BEGIN() { - // Build unit test - TestBuild *const testBld = testBldNew(pathRepo, pathTest, vm, vmId, moduleName, test, scale, logLevel, logTime, timeZone); - testBldUnit(testBld); + // Create the data path + const Storage *const storageHrnId = storagePosixNewP(strNewFmt("%s/data-%u", strZ(pathTest), vmId), .write = true); + cmdTestExecLog = storagePathP(storageHrnId, STRDEF("exec.log")); - // Remove and recreate the test path - const Storage *const storageTestId = storagePosixNewP( - strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true); - const char *const permReset = zNewFmt("chmod -R 777 %s", strZ(storagePathP(storageTestId, NULL))); + cmdTestPathCreate(storageHrnId, NULL); - if (system(permReset) != 0) - THROW_FMT(ExecuteError, "unable to execute: %s", permReset); + // Copy the source repository if requested (otherwise defaults to source code repository) + const String *pathRepoCopy = pathRepo; - storagePathRemoveP(storageTestId, NULL, .recurse = true); - storagePathCreateP(storageTestId, NULL, .mode = 0770); - - // Meson setup - const String *const pathUnit = strNewFmt("%s/unit-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)); - const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit)); - - if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild)))) + if (repoCopy) { - const char *const mesonSetup = zNewFmt( - "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s", strZ(pathUnitBuild), strZ(pathUnit)); + pathRepoCopy = strNewFmt("%s/repo", strZ(pathTest)); + const Storage *const storageRepoCopy = storagePosixNewP(pathRepoCopy, .write = true); - if (system(mesonSetup) != 0) - THROW_FMT(ExecuteError, "unable to execute: %s", mesonSetup); + LOG_DETAIL_FMT("sync repo to %s", strZ(pathRepoCopy)); + storagePathCreateP(storageRepoCopy, NULL, .mode = 0770); + + cmdTestExec( + strNewFmt( + "git -C %s ls-files -c --others --exclude-standard | rsync -LtW --files-from=- %s/ %s", strZ(pathRepo), + strZ(pathRepo), strZ(pathRepoCopy))); } - // Ninja build - const char *const ninjaBuild = zNewFmt("ninja -C %s", strZ(pathUnitBuild)); + // Build code (??? better to do this only when it is needed) + cmdTestExec(strNewFmt("%s/build/none/src/build-code postgres %s/extra", strZ(pathTest), strZ(pathRepoCopy))); - if (system(ninjaBuild) != 0) - THROW_FMT(ExecuteError, "unable to execute: %s", ninjaBuild); + // Build test list + const TestDef testDef = testDefParse(storagePosixNewP(pathRepoCopy)); + StringList *const moduleList = strLstNew(); + bool binRequired = false; - // Unit test - const char *const unitTest = zNewFmt("%s/test-unit", strZ(pathUnitBuild)); + if (strLstEmpty(moduleFilterList)) + { + StringList *const moduleFilterListEmpty = strLstNew(); + strLstAddZ(moduleFilterListEmpty, ""); - if (system(unitTest) != 0) - THROW_FMT(ExecuteError, "unable to execute: %s", unitTest); + moduleFilterList = moduleFilterListEmpty; + } + + for (unsigned int moduleFilterIdx = 0; moduleFilterIdx < strLstSize(moduleFilterList); moduleFilterIdx++) + { + const String *const moduleFilter = strLstGet(moduleFilterList, moduleFilterIdx); + + if (strEmpty(moduleFilter) || strEndsWithZ(moduleFilter, "/")) + { + bool found = false; + + for (unsigned int moduleIdx = 0; moduleIdx < lstSize(testDef.moduleList); moduleIdx++) + { + const TestDefModule *const module = lstGet(testDef.moduleList, moduleIdx); + + // ??? These test types don't run yet + if (module->flag != NULL || module->containerRequired) + continue; + + if (strEmpty(moduleFilter) || strBeginsWith(module->name, moduleFilter)) + { + strLstAddIfMissing(moduleList, module->name); + found = true; + + if (module->binRequired) + binRequired = true; + } + } + + if (!found) + THROW_FMT(ParamInvalidError, "'%s' prefix does not match any tests", strZ(moduleFilter)); + } + else + { + const TestDefModule *const module = lstFind(testDef.moduleList, &moduleFilter); + + if (module == NULL) + THROW_FMT(ParamInvalidError, "'%s' is not a valid test", strZ(moduleFilter)); + + strLstAddIfMissing(moduleList, module->name); + } + } + + // Build pgbackrest exe + if (binRequired) + { + LOG_INFO("build pgbackrest"); + cmdTestExec(strNewFmt("ninja -C %s/build/none src/pgbackrest", strZ(pathTest))); + } + + // Process test list + unsigned int errorTotal = 0; + + for (unsigned int moduleIdx = 0; moduleIdx < strLstSize(moduleList); moduleIdx++) + { + const String *const moduleName = strLstGet(moduleList, moduleIdx); + const TestDefModule *const module = lstFind(testDef.moduleList, &moduleName); + CHECK(AssertError, module != NULL, "unable to find module"); + + TRY_BEGIN() + { + // Build unit test + const TimeMSec buildTimeBegin = timeMSec(); + TestBuild *const testBld = testBldNew( + pathRepoCopy, pathTest, vm, vmId, module, test, scale, logLevel, logTime, timeZone); + testBldUnit(testBld); + + // Create the test path + const Storage *const storageTestId = storagePosixNewP( + strNewFmt("%s/test-%u", strZ(testBldPathTest(testBld)), testBldVmId(testBld)), .write = true); + + cmdTestPathCreate(storageTestId, NULL); + + // Meson setup + const String *const pathUnit = strNewFmt("%s/unit-%u", strZ(pathTest), vmId); + const String *const pathUnitBuild = strNewFmt("%s/build", strZ(pathUnit)); + + if (!storageExistsP(testBldStorageTest(testBld), strNewFmt("%s/build.ninja", strZ(pathUnitBuild)))) + { + LOG_DETAIL("meson setup"); + cmdTestExec( + strNewFmt( + "meson setup -Dwerror=true -Dfatal-errors=true -Dbuildtype=debug %s %s", strZ(pathUnitBuild), + strZ(pathUnit))); + } + + // Ninja build + cmdTestExec(strNewFmt("ninja -C %s", strZ(pathUnitBuild))); + const TimeMSec buildTimeEnd = timeMSec(); + + // Unit test + const TimeMSec runTimeBegin = timeMSec(); + cmdTestExec(strNewFmt("%s/test-unit", strZ(pathUnitBuild))); + const TimeMSec runTimeEnd = timeMSec(); + + LOG_INFO_FMT( + "test unit %s (bld=%.3fs, run=%.3fs)", strZ(moduleName), + (double)(buildTimeEnd - buildTimeBegin) / (double)MSEC_PER_SEC, + (double)(runTimeEnd - runTimeBegin) / (double)MSEC_PER_SEC); + } + CATCH_ANY() + { + LOG_INFO_FMT("test unit %s", strZ(moduleName)); + LOG_ERROR(errorCode(), errorMessage()); + errorTotal++; + } + TRY_END(); + } + + // Return error + if (errorTotal > 0) + THROW_FMT(CommandError, "%u test failure(s)", errorTotal); } MEM_CONTEXT_TEMP_END(); diff --git a/test/src/command/test/test.h b/test/src/command/test/test.h index 91e04f6f2..793b06c64 100644 --- a/test/src/command/test/test.h +++ b/test/src/command/test/test.h @@ -7,13 +7,13 @@ Perform a test. #define TEST_COMMAND_TEST_TEST_H #include "common/logLevel.h" -#include "common/type/string.h" +#include "common/type/stringList.h" /*********************************************************************************************************************************** Functions ***********************************************************************************************************************************/ void cmdTest( - const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const String *moduleName, - unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone); + const String *pathRepo, const String *pathTest, const String *const vm, unsigned int vmId, const StringList *moduleFilterList, + unsigned int test, uint64_t scale, LogLevel logLevel, bool logTime, const String *timeZone, bool repoCopy); #endif diff --git a/test/src/main.c b/test/src/main.c index 84ee8120f..f038c34d7 100644 --- a/test/src/main.c +++ b/test/src/main.c @@ -72,10 +72,9 @@ main(int argListSize, const char *argList[]) { cmdTest( cfgOptionStr(cfgOptRepoPath), cfgOptionStr(cfgOptTestPath), cfgOptionStr(cfgOptVm), - cfgOptionUInt(cfgOptVmId), cfgOptionStr(cfgOptModule), - cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, cfgOptionUInt64(cfgOptScale), - logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)), cfgOptionBool(cfgOptLogTimestamp), - cfgOptionStrNull(cfgOptTz)); + cfgOptionUInt(cfgOptVmId), cfgCommandParam(), cfgOptionTest(cfgOptTest) ? cfgOptionUInt(cfgOptTest) : 0, + cfgOptionUInt64(cfgOptScale), logLevelEnum(cfgOptionStrId(cfgOptLogLevelTest)), + cfgOptionBool(cfgOptLogTimestamp), cfgOptionStrNull(cfgOptTz), cfgOptionBool(cfgOptRepoCopy)); break; }
Unit test module to be tested.
Make a copy of the code repository for testing so changes to the code repository do not affect tests.
Path to the original code repository or a copy.