1
0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2024-12-12 10:04:14 +02:00
pgbackrest/test/define.yaml

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

1047 lines
34 KiB
YAML
Raw Normal View History

# **********************************************************************************************************************************
# Test Definition
#
# Contains definitions for all unit and integration tests.
#
# The definitions consist of modules and tests. Modules group together tests that are related and allow common settings for all the
# tests. Every module and test must have a name. Tests must also have a total. If this total does not match the actual number of
# runs in a test then an error will be thrown.
#
# Most options can be set for modules and tests (test option will override module option if both are set):
# * db - determines if the test will be run against multiple db versions
# * define - defines for C code (will also be applied to the test harness)
# * binReq - is the pgbackrest binary required for this test?
# * containerReq - is this test required to run in a container?
#
# Some options are unique to tests:
# * coverage - a list of code modules that the test provides coverage for. A code module may be covered by multiple tests. That
# means you must run all the tests by providing the --run option multiple times to get full coverage on the code module. If
# a code module contains only data it should be marked noCode (e.g. - help/help.auto: noCode). If a code module is included
# in another code module it should be marked as included (e.g. - md5.vendor: included).
# * feature - Defines a feature that is now available in the harness. For example, the "error" feature defines HRN_FEATURE_ERROR
# used to automatically check for errors in later tests. The common/error test is not able to access this error handling
# because it is used to implement the error handling, so it must do error testing in a more primitive way.
# * harness - Adds a harness module that contains functions to aid in testing. For example, the "log" harness includes the
# common/harnessLog module to aid in expect log testing.
# * shim - list of modules that are shimmed in the harness. This allows the harness to access static elements of the module to
# provide additional services for unit testing. A shim can have a 'function' list. In this case the listed functions in the
# C module will be appended with _SHIMMED and an implementation with the same function signature must be provided in the
# harness. Generally speaking this function will default to calling the original function, but after some initialization the
# shim may implement other logic that is useful for unit testing.
# * depend - Source modules that this test depends on that have not already been included in prior tests via coverage. Ideally
# this option would never be used because it is essentially documenting cross-dependencies in the code.
# * total - total runs in the test
# * vm - VMs that the test will be run on
# * include - modules to include directly into test.c (all files in coverage are automatically included)
# This is useful when a module's internal data needs to be manipulated for testing but no coverage is added by the test.
# **********************************************************************************************************************************
# **********************************************************************************************************************************
# Unit tests
# **********************************************************************************************************************************
unit:
# ********************************************************************************************************************************
- name: common
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: error
total: 9
feature: error
harness:
name: error
shim:
common/error/error: ~
coverage:
- common/error/error
- common/error/error.auto: noCode
depend:
- common/stackTrace
# ----------------------------------------------------------------------------------------------------------------------------
- name: stack-trace
total: 4
feature: stackTrace
harness:
name: stackTrace
shim:
common/stackTrace:
function:
- stackTraceBackCallback
coverage:
- common/stackTrace
depend:
- common/debug
- common/type/stringStatic
# ----------------------------------------------------------------------------------------------------------------------------
- name: assert-off
total: 1
define: -DNDEBUG
coverage:
- common/assert: noCode
# ----------------------------------------------------------------------------------------------------------------------------
- name: debug-off
total: 1
define: -DNDEBUG
# ----------------------------------------------------------------------------------------------------------------------------
- name: assert-on
feature: assert
total: 1
coverage:
- common/assert: noCode
# ----------------------------------------------------------------------------------------------------------------------------
- name: mem-context
total: 8
feature: memContext
coverage:
- common/memContext
depend:
- common/type/convert
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-convert
total: 12
coverage:
- common/type/convert
# ----------------------------------------------------------------------------------------------------------------------------
- name: time
total: 3
harness:
name: time
shim:
common/time:
function:
- timeMSec
coverage:
- common/time
# ----------------------------------------------------------------------------------------------------------------------------
- name: encode
total: 3
coverage:
- common/encode
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-object
total: 1
coverage:
- common/type/object
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-blob
total: 1
coverage:
- common/type/blob
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-string
total: 26
feature: string
coverage:
- build/common/string
- common/type/string: included
- common/type/stringZ
Add StringId type. It is often useful to represent identifiers as strings when they cannot easily be represented as an enum/integer, e.g. because they are distributed among a number of unrelated modules or need to be passed to remote processes. Strings are also more helpful in debugging since they can be recognized without cross-referencing the source. However, strings are awkward to work with in C since they cannot be directly used in switch statements leading to less efficient if-else structures. A StringId encodes a short string into an integer so it can be used in switch statements but may also be readily converted back into a string for debugging purposes. StringIds may also be suitable for matching user input providing the strings are short enough. This patch includes a sample of StringId usage by converting protocol commands to StringIds. There are many other possible use cases. To list a few: * All "types" in storage, filters. IO , etc. These types are primarily for identification and debugging so they fit well with this model. * MemContext names would work well as StringIds since these are entirely for debugging. * Option values could be represented as StringIds which would mean we could remove the functions that convert strings to enums, e.g. CipherType. * There are a number of places where enums need to be converted back to strings for logging/debugging purposes. An example is protocolParallelJobToConstZ. If ProtocolParallelJobState were defined as: typedef enum { protocolParallelJobStatePending = STRID5("pend", ...), protocolParallelJobStateRunning = STRID5("run", ...), protocolParallelJobStateDone = STRID5("done", ...), } ProtocolParallelJobState; then protocolParallelJobToConstZ() could be replaced with strIdToZ(). This also applies to many enums that we don't covert to strings for logging, such as CipherMode. As an example of usage, convert all protocol commands from strings to StringIds.
2021-04-20 21:22:42 +02:00
- common/type/stringId
- common/type/stringList
depend:
- common/type/buffer
- common/type/keyValue
- common/type/list
- common/type/variant
- common/type/variantList
# ----------------------------------------------------------------------------------------------------------------------------
- name: error-retry
total: 1
harness:
name: errorRetry
shim:
common/error/retry:
function:
- errRetryMessage
coverage:
- common/error/retry
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-list
total: 4
coverage:
- common/type/list
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-buffer
total: 5
coverage:
- common/type/buffer
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-variant
total: 12
coverage:
- common/type/variant
- common/type/variantList
# ----------------------------------------------------------------------------------------------------------------------------
- name: reg-exp
total: 3
coverage:
- build/common/regExp
- common/regExp: included
# ----------------------------------------------------------------------------------------------------------------------------
- name: log
total: 5
feature: log
harness:
name: log
shim:
common/log: ~
coverage:
- common/log
# ----------------------------------------------------------------------------------------------------------------------------
- name: debug-on
total: 4
feature: debug
coverage:
- common/debug
- common/type/stringStatic
# ----------------------------------------------------------------------------------------------------------------------------
- name: fork
total: 1
coverage:
- common/fork
# ----------------------------------------------------------------------------------------------------------------------------
- name: wait
total: 1
coverage:
- common/wait
2018-08-09 14:06:23 +02:00
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-json
total: 2
2018-08-09 14:06:23 +02:00
coverage:
- common/type/json
2018-08-09 14:06:23 +02:00
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-key-value
total: 2
coverage:
- common/type/keyValue
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-xml
total: 1
coverage:
- build/common/xml
- common/type/xml: included
# ----------------------------------------------------------------------------------------------------------------------------
- name: stat
total: 1
feature: STAT
coverage:
- common/stat
# ----------------------------------------------------------------------------------------------------------------------------
- name: user
total: 1
coverage:
- common/user
# ----------------------------------------------------------------------------------------------------------------------------
- name: io
total: 6
feature: IO
harness: pack
coverage:
- common/io/bufferRead
- common/io/bufferWrite
- common/io/fd
- common/io/fdRead
- common/io/fdWrite
- common/io/filter/buffer
- common/io/filter/filter
- common/io/filter/group
- common/io/filter/sink
- common/io/filter/size
- common/io/io
- common/io/limitRead
- common/io/read
- common/io/write
depend:
- common/type/pack
# ----------------------------------------------------------------------------------------------------------------------------
- name: type-pack
total: 1
coverage:
- common/type/pack
# ----------------------------------------------------------------------------------------------------------------------------
- name: compress
total: 5
coverage:
- common/compress/bz2/common
- common/compress/bz2/compress
- common/compress/bz2/decompress
- common/compress/gz/common
- common/compress/gz/compress
- common/compress/gz/decompress
- common/compress/lz4/common
- common/compress/lz4/compress
- common/compress/lz4/decompress
- common/compress/zst/common
- common/compress/zst/compress
- common/compress/zst/decompress
- common/compress/helper
depend:
- storage/posix/read
- storage/posix/storage
- storage/posix/write
- storage/iterator
- storage/list
- storage/read
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: crypto
total: 4
feature: STORAGE
harness: storage
coverage:
- common/crypto/cipherBlock
- common/crypto/common
- common/crypto/hash
- common/crypto/md5.vendor: included
- common/crypto/xxhash
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-tls
total: 6
feature: SOCKET
harness: server
2023-05-13 18:16:16 +02:00
harness:
name: socket
shim:
common/io/socket/client:
function:
- sckClientOpen
- sckClientOpenWait
coverage:
- common/io/client
- common/io/server
- common/io/session
- common/io/tls/client
- common/io/tls/common
- common/io/tls/server
- common/io/tls/session
- common/io/socket/address
- common/io/socket/client
- common/io/socket/common
- common/io/socket/server
- common/io/socket/session
include:
- common/io/fdRead
- common/io/read
# ----------------------------------------------------------------------------------------------------------------------------
- name: io-http
total: 6
coverage:
- common/io/http/client
- common/io/http/common
- common/io/http/header
- common/io/http/query
- common/io/http/request
- common/io/http/response
- common/io/http/session
- common/io/http/url
include:
- common/io/socket/address
# ----------------------------------------------------------------------------------------------------------------------------
- name: exec
total: 2
coverage:
- build/common/exec
# ----------------------------------------------------------------------------------------------------------------------------
- name: ini
total: 2
coverage:
- common/ini
# ----------------------------------------------------------------------------------------------------------------------------
- name: lock
total: 3
harness:
name: config
shim:
config/load: ~
harness:
name: lock
shim:
common/lock:
function:
- lockInit
coverage:
- common/lock
depend:
- common/lock
- config/common
- config/config
- config/parse
- config/exec
- config/load
- config/protocol
- command/command
- postgres/interface
- postgres/interface/crc32
- postgres/interface/page
- protocol/client
- protocol/command
- protocol/helper
- protocol/server
- storage/helper
- storage/remote/read
- storage/remote/protocol
- storage/remote/storage
- storage/remote/write
# ********************************************************************************************************************************
- name: protocol
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: protocol
total: 8
harness:
name: protocol
shim:
protocol/helper:
function:
- protocolLocalExec
- protocolRemoteExec
containerReq: true
binReq: true
coverage:
- protocol/client
- protocol/command
- protocol/helper
- protocol/parallel
- protocol/parallelJob
- protocol/server
include:
- build/common/exec
# ********************************************************************************************************************************
- name: config
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: parse
total: 6
coverage:
- config/common
- config/config
- config/parse
- config/parse.auto: noCode
# ----------------------------------------------------------------------------------------------------------------------------
- name: load
total: 4
coverage:
- config/load
include:
- common/io/socket/common
# ----------------------------------------------------------------------------------------------------------------------------
- name: exec
total: 1
coverage:
- config/exec
# ----------------------------------------------------------------------------------------------------------------------------
- name: protocol
total: 1
coverage:
- config/protocol
# ********************************************************************************************************************************
- name: storage
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: posix
total: 23
coverage:
- storage/cifs/helper
- storage/cifs/storage
- storage/posix/read
- storage/posix/storage
- storage/posix/write
- storage/helper
- storage/iterator
- storage/list
- storage/read
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: remote
total: 10
coverage:
- storage/remote/read
- storage/remote/protocol
- storage/remote/storage
- storage/remote/write
- storage/helper
- storage/storage
include:
- storage/read
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: azure
total: 3
coverage:
- storage/azure/helper
- storage/azure/read
- storage/azure/storage
- storage/azure/write
include:
- storage/helper
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: gcs
total: 3
coverage:
- storage/gcs/helper
- storage/gcs/read
- storage/gcs/storage
- storage/gcs/write
include:
- storage/helper
- storage/storage
- storage/write
# ----------------------------------------------------------------------------------------------------------------------------
- name: s3
total: 2
coverage:
- storage/s3/helper
- storage/s3/read
- storage/s3/storage
- storage/s3/write
- storage/helper
include:
- storage/storage
- storage/write
2023-05-13 18:16:16 +02:00
# ----------------------------------------------------------------------------------------------------------------------------
- name: sftp
total: 19
harness:
name: libSsh2
integration: false
harness:
name: fd
shim:
common/io/fd:
function:
- fdReady
2023-05-13 18:16:16 +02:00
coverage:
- storage/sftp/helper
- storage/sftp/read
- storage/sftp/storage
- storage/sftp/write
include:
- storage/helper
- storage/read
- storage/storage
- storage/write
# ********************************************************************************************************************************
- name: postgres
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: client
total: 1
harness:
name: pq
integration: false
coverage:
- postgres/client
# ----------------------------------------------------------------------------------------------------------------------------
- name: interface
total: 10
harness: postgres
coverage:
- postgres/interface
- postgres/interface/crc32
- postgres/interface/page
# ********************************************************************************************************************************
- name: build
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: common
total: 2
coverage:
- build/common/render
- build/common/yaml
# ----------------------------------------------------------------------------------------------------------------------------
- name: config
total: 1
coverage:
- build/config/parse
- build/config/render
# ----------------------------------------------------------------------------------------------------------------------------
- name: error
total: 1
coverage:
- build/error/parse
- build/error/render
# ----------------------------------------------------------------------------------------------------------------------------
- name: help
total: 2
coverage:
- build/help/parse
- build/help/render
# ----------------------------------------------------------------------------------------------------------------------------
- name: postgres
total: 1
coverage:
- build/postgres/parse
- build/postgres/render
# ********************************************************************************************************************************
- name: test
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: test
total: 2
coverage:
- test/command/test/build
- test/command/test/define
- test/command/test/test
# ********************************************************************************************************************************
- name: info
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: info
total: 3
harness: info
coverage:
- info/info
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-pg
total: 2
coverage:
- info/infoPg
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-archive
total: 2
coverage:
- info/infoArchive
# ----------------------------------------------------------------------------------------------------------------------------
- name: manifest
total: 6
harness:
name: manifest
shim:
info/manifest: ~
coverage:
- info/manifest
depend:
- command/backup/common
# ----------------------------------------------------------------------------------------------------------------------------
- name: info-backup
total: 3
coverage:
- info/infoBackup
# ********************************************************************************************************************************
- name: db
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: db
total: 3
containerReq: true
binReq: true
coverage:
- db/db
- db/helper
- db/protocol
# ********************************************************************************************************************************
- name: command
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: control
total: 4
coverage:
- command/control/common
- command/control/start
- command/control/stop
# ----------------------------------------------------------------------------------------------------------------------------
- name: annotate
total: 1
coverage:
- command/annotate/annotate
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-common
total: 9
coverage:
- command/archive/common
- command/archive/find
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-get
total: 3
binReq: true
coverage:
- command/archive/common
- command/archive/get/file
- command/archive/get/get
- command/archive/get/protocol
# ----------------------------------------------------------------------------------------------------------------------------
- name: archive-push
total: 4
binReq: true
coverage:
- command/archive/common
- command/archive/push/file
- command/archive/push/protocol
- command/archive/push/push
# ----------------------------------------------------------------------------------------------------------------------------
- name: stanza
total: 4
coverage:
- command/stanza/common
- command/stanza/create
- command/stanza/upgrade
- command/stanza/delete
depend:
- command/check/common
# ----------------------------------------------------------------------------------------------------------------------------
- name: check
total: 4
coverage:
- command/check/common
- command/check/check
- command/check/report
# ----------------------------------------------------------------------------------------------------------------------------
- name: command
total: 1
coverage:
- command/command
# ----------------------------------------------------------------------------------------------------------------------------
- name: expire
total: 8
coverage:
- command/expire/expire
include:
- info/infoBackup
# ----------------------------------------------------------------------------------------------------------------------------
- name: help
total: 4
coverage:
- command/help/help
- command/help/help.auto: noCode
include:
- build/help/render
# ----------------------------------------------------------------------------------------------------------------------------
- name: info
total: 3
coverage:
- command/info/info
# ----------------------------------------------------------------------------------------------------------------------------
- name: backup
total: 12
harness:
name: backup
integration: false
shim:
command/backup/backup:
function:
- backupProcess
harness:
name: blockIncr
shim:
command/restore/blockDelta: ~
coverage:
- command/backup/backup
- command/backup/blockIncr
- command/backup/blockMap
- command/backup/common
- command/backup/file
- command/backup/pageChecksum
- command/backup/protocol
- command/restore/blockDelta
include:
- info/info
- info/manifest
- storage/storage
# ----------------------------------------------------------------------------------------------------------------------------
- name: restore
total: 15
coverage:
- command/restore/blockChecksum
- command/restore/blockDelta
- command/restore/file
- command/restore/protocol
- command/restore/restore
include:
- common/user
- info/infoBackup
- info/manifest
# ----------------------------------------------------------------------------------------------------------------------------
- name: manifest
total: 1
include:
- command/manifest/manifest
# ----------------------------------------------------------------------------------------------------------------------------
- name: repo
total: 4
coverage:
- command/repo/common
- command/repo/get
- command/repo/ls
- command/repo/put
- command/repo/rm
include:
# command/repo/create is currently for testing purposes only so coverage is not provided except in integration. In the
# future this will probably be rolled into a custom object store server implementation.
- command/repo/create
# ----------------------------------------------------------------------------------------------------------------------------
- name: verify
total: 12
coverage:
- command/verify/file
- command/verify/protocol
- command/verify/verify
# ----------------------------------------------------------------------------------------------------------------------------
- name: local
total: 1
coverage:
- command/local/local
# ----------------------------------------------------------------------------------------------------------------------------
- name: remote
total: 1
coverage:
- command/remote/remote
# ----------------------------------------------------------------------------------------------------------------------------
- name: exit
total: 3
coverage:
- command/exit
# ----------------------------------------------------------------------------------------------------------------------------
- name: server
total: 2
coverage:
- command/server/ping
- command/server/server
# ********************************************************************************************************************************
- name: doc
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: build
total: 1
coverage:
- doc/command/build/build
2024-01-23 14:56:26 +02:00
- doc/command/build/man
- doc/command/build/reference
# **********************************************************************************************************************************
# Integration tests
#
# Integration tests are not run in a container. They are expected to create their own containers since most integration runs will
# create more than one. For this reason each run is executed individually.
# **********************************************************************************************************************************
integration:
# ********************************************************************************************************************************
- name: integration
db: true
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: all
Simplify test matrix for real/all tests. Test matrices were previously simplified for the mock/* tests (e.g. d4410611, d489eb87) but not for real/all since the rules for which tests would run with which options was extremely complex. This only got more complex when new compression formats were added. Because the loop-generated matrix was so large, mosts tests were skipped for most option combinations following arcane logic which was nearly impossible to decipher even when reading the code, and completely impossible from the test.pl interface. As a consequence, important tests got excluded. For example, backup from standby was excluded for most versions of PostgreSQL because it was only run once per distro, against the latest version to be included in that distro. Simplify the tests by having a single run per PostgreSQL version and vary test parameters according to the capabilities of each version and the underlying distro. So, ZST testing is based on whether the distro supports ZST. Every test is run for each set of parameters based on the capabilities of the PostgreSQL version, e.g. backup from standby is not attempted on versions that don't support it. Note that since more tests are running the overall time to run the mock/all tests has increased by about 20-25%. Some time may be saved my removing tests that are adequately covered by unit tests but that should the subject of another commit. Another option would be to limit some non version-specific tests to a single, well defined version of PostgreSQL, .e.g the version that is run by expect tests, currently 9.6. The motivation for this refactor is that new storage drivers are coming and the loop-generated test matrix simply was not up to the task of adding them. The following is an example of the new test log (note longer runtime of each test): module=real, test=all, run=1, pg-version=10 (106.91s) module=real, test=all, run=1, pg-version=9.5 (151.09s) module=real, test=all, run=1, pg-version=9.2 (123.11s) module=real, test=all, run=1, pg-version=9.1 (129s) vs. the old test log (sub-second tests were skipped entirely): module=real, test=all, run=2, pg-version=10 (0.31s) module=real, test=all, run=3, pg-version=10 (0.26s) module=real, test=all, run=4, pg-version=10 (60.39s) module=real, test=all, run=1, pg-version=10 (69.12s) module=real, test=all, run=6, pg-version=10 (34s) module=real, test=all, run=5, pg-version=10 (42.75s) module=real, test=all, run=2, pg-version=9.5 (0.21s) module=real, test=all, run=3, pg-version=9.5 (0.21s) module=real, test=all, run=4, pg-version=9.5 (0.21s) module=real, test=all, run=5, pg-version=9.5 (0.26s) module=real, test=all, run=6, pg-version=9.5 (0.21s) module=real, test=all, run=1, pg-version=9.2 (72.78s) module=real, test=all, run=2, pg-version=9.2 (0.26s) module=real, test=all, run=3, pg-version=9.2 (0.31s) module=real, test=all, run=4, pg-version=9.2 (0.21s) module=real, test=all, run=5, pg-version=9.2 (0.21s) module=real, test=all, run=6, pg-version=9.2 (0.21s) module=real, test=all, run=1, pg-version=9.5 (88.41s) module=real, test=all, run=2, pg-version=9.1 (0.21s) module=real, test=all, run=3, pg-version=9.1 (0.26s) module=real, test=all, run=4, pg-version=9.1 (0.21s) module=real, test=all, run=5, pg-version=9.1 (0.31s) module=real, test=all, run=6, pg-version=9.1 (0.26s) module=real, test=all, run=1, pg-version=9.1 (72.4s)
2020-06-23 19:44:29 +02:00
total: 1
harness: host
# **********************************************************************************************************************************
# Performance tests
#
# Performance tests run in a single container but are more like integration tests than unit tests since they call the pgbackrest
# executable directly.
# **********************************************************************************************************************************
performance:
# ********************************************************************************************************************************
- name: performance
test:
# ----------------------------------------------------------------------------------------------------------------------------
- name: type
total: 6
# ----------------------------------------------------------------------------------------------------------------------------
- name: storage
total: 2
include:
- storage/helper