You've already forked pgbackrest
mirror of
https://github.com/pgbackrest/pgbackrest.git
synced 2025-07-15 01:04:37 +02:00
Add general-purpose statistics collector.
Currently each module that needs to collect statistics implements custom code to do so. This is cumbersome. Create a general purpose module for collecting and reporting statistics. Statistics are output in the log at detail level, but there are other uses they could be put to eventually. No new functionality is added. This is just a drop-in replacement for the current statistics, with the advantage of being more flexible. The new stats are slower because they involve a list lookup, but performance testing shows stats can be updated at about 40,000/ms which seems fast enough for our purposes.
This commit is contained in:
David Steele
GitHub
@ -121,6 +121,14 @@
|
||||
<p>Add support for <id>HTTP/1.0</id>.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-reviewer id="cynthia.shang"/>
|
||||
</release-item-contributor-list>
|
||||
|
||||
<p>Add general-purpose statistics collector.</p>
|
||||
</release-item>
|
||||
|
||||
<release-item>
|
||||
<release-item-contributor-list>
|
||||
<release-item-reviewer id="stephen.frost"/>
|
||||
|
||||
@ -97,6 +97,7 @@ SRCS = \
|
||||
common/memContext.c \
|
||||
common/regExp.c \
|
||||
common/stackTrace.c \
|
||||
common/stat.c \
|
||||
common/time.c \
|
||||
common/type/buffer.c \
|
||||
common/type/convert.c \
|
||||
|
||||
@ -7,12 +7,11 @@ Common Command Routines
|
||||
#include <string.h>
|
||||
|
||||
#include "common/debug.h"
|
||||
#include "common/io/http/client.h"
|
||||
#include "common/io/socket/client.h"
|
||||
#include "common/io/tls/client.h"
|
||||
#include "common/log.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/time.h"
|
||||
#include "common/type/json.h"
|
||||
#include "config/config.h"
|
||||
#include "version.h"
|
||||
|
||||
@ -181,23 +180,11 @@ cmdEnd(int code, const String *errorMessage)
|
||||
{
|
||||
MEM_CONTEXT_TEMP_BEGIN()
|
||||
{
|
||||
// Log socket statistics
|
||||
String *sckClientStat = sckClientStatStr();
|
||||
// Output statistics if there are any
|
||||
const KeyValue *statKv = statToKv();
|
||||
|
||||
if (sckClientStat != NULL)
|
||||
LOG_DETAIL(strZ(sckClientStat));
|
||||
|
||||
// Log tls statistics
|
||||
String *tlsClientStat = tlsClientStatStr();
|
||||
|
||||
if (tlsClientStat != NULL)
|
||||
LOG_DETAIL(strZ(tlsClientStat));
|
||||
|
||||
// Log http statistics
|
||||
String *httpClientStat = httpClientStatStr();
|
||||
|
||||
if (httpClientStat != NULL)
|
||||
LOG_INFO(strZ(httpClientStat));
|
||||
if (varLstSize(kvKeyList(statKv)) > 0)
|
||||
LOG_DETAIL_FMT("statistics: %s", strZ(jsonFromKv(statKv)));
|
||||
|
||||
// Basic info on command end
|
||||
String *info = strNewFmt("%s command end: ", strZ(cfgCommandRoleName()));
|
||||
|
||||
@ -7,12 +7,17 @@ HTTP Client
|
||||
#include "common/io/client.h"
|
||||
#include "common/io/http/client.h"
|
||||
#include "common/log.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/object.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
HttpClientStat httpClientStat;
|
||||
STRING_EXTERN(HTTP_STAT_CLIENT_STR, HTTP_STAT_CLIENT);
|
||||
STRING_EXTERN(HTTP_STAT_CLOSE_STR, HTTP_STAT_CLOSE);
|
||||
STRING_EXTERN(HTTP_STAT_REQUEST_STR, HTTP_STAT_REQUEST);
|
||||
STRING_EXTERN(HTTP_STAT_RETRY_STR, HTTP_STAT_RETRY);
|
||||
STRING_EXTERN(HTTP_STAT_SESSION_STR, HTTP_STAT_SESSION);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
@ -53,7 +58,7 @@ httpClientNew(IoClient *ioClient, TimeMSec timeout)
|
||||
.sessionReuseList = lstNewP(sizeof(HttpSession *)),
|
||||
};
|
||||
|
||||
httpClientStat.object++;
|
||||
statInc(HTTP_STAT_CLIENT_STR);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
@ -86,7 +91,7 @@ httpClientOpen(HttpClient *this)
|
||||
else
|
||||
{
|
||||
result = httpSessionNew(this, ioClientOpen(this->ioClient));
|
||||
httpClientStat.session++;
|
||||
statInc(HTTP_STAT_SESSION_STR);
|
||||
}
|
||||
|
||||
FUNCTION_LOG_RETURN(HTTP_SESSION, result);
|
||||
@ -118,22 +123,3 @@ httpClientToLog(const HttpClient *this)
|
||||
"{ioClient: %s, reusable: %u, timeout: %" PRIu64"}", strZ(ioClientToLog(this->ioClient)), lstSize(this->sessionReuseList),
|
||||
this->timeout);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
String *
|
||||
httpClientStatStr(void)
|
||||
{
|
||||
FUNCTION_TEST_VOID();
|
||||
|
||||
String *result = NULL;
|
||||
|
||||
if (httpClientStat.object > 0)
|
||||
{
|
||||
result = strNewFmt(
|
||||
"http statistics: objects %" PRIu64 ", sessions %" PRIu64 ", requests %" PRIu64 ", retries %" PRIu64
|
||||
", closes %" PRIu64,
|
||||
httpClientStat.object, httpClientStat.session, httpClientStat.request, httpClientStat.retry, httpClientStat.close);
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RETURN(result);
|
||||
}
|
||||
|
||||
@ -29,18 +29,18 @@ typedef struct HttpClient HttpClient;
|
||||
#include "common/time.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct HttpClientStat
|
||||
{
|
||||
uint64_t object; // Objects created
|
||||
uint64_t session; // Sessions created
|
||||
uint64_t request; // Requests (i.e. calls to httpRequestNew())
|
||||
uint64_t retry; // Request retries
|
||||
uint64_t close; // Closes forced by server
|
||||
} HttpClientStat;
|
||||
|
||||
extern HttpClientStat httpClientStat;
|
||||
#define HTTP_STAT_CLIENT "http.client" // Clients created
|
||||
STRING_DECLARE(HTTP_STAT_CLIENT_STR);
|
||||
#define HTTP_STAT_CLOSE "http.close" // Closes forced by server
|
||||
STRING_DECLARE(HTTP_STAT_CLOSE_STR);
|
||||
#define HTTP_STAT_REQUEST "http.request" // Requests (i.e. calls to httpRequestNew())
|
||||
STRING_DECLARE(HTTP_STAT_REQUEST_STR);
|
||||
#define HTTP_STAT_RETRY "http.retry" // Request retries
|
||||
STRING_DECLARE(HTTP_STAT_RETRY_STR);
|
||||
#define HTTP_STAT_SESSION "http.session" // Sessions created
|
||||
STRING_DECLARE(HTTP_STAT_SESSION_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
@ -56,9 +56,6 @@ HttpSession *httpClientOpen(HttpClient *this);
|
||||
// Request/response finished cleanly so session can be reused
|
||||
void httpClientReuse(HttpClient *this, HttpSession *session);
|
||||
|
||||
// Format statistics to a string
|
||||
String *httpClientStatStr(void);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Getters/Setters
|
||||
***********************************************************************************************************************************/
|
||||
|
||||
@ -7,6 +7,7 @@ HTTP Request
|
||||
#include "common/io/http/common.h"
|
||||
#include "common/io/http/request.h"
|
||||
#include "common/log.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/wait.h"
|
||||
#include "version.h"
|
||||
@ -161,7 +162,7 @@ httpRequestProcess(HttpRequest *this, bool waitForResponse, bool contentCache)
|
||||
LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage());
|
||||
retry = true;
|
||||
|
||||
httpClientStat.retry++;
|
||||
statInc(HTTP_STAT_RETRY_STR);
|
||||
}
|
||||
else
|
||||
RETHROW();
|
||||
@ -213,7 +214,7 @@ httpRequestNew(HttpClient *client, const String *verb, const String *uri, HttpRe
|
||||
|
||||
// Send the request
|
||||
httpRequestProcess(this, false, false);
|
||||
httpClientStat.request++;
|
||||
statInc(HTTP_STAT_REQUEST_STR);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
|
||||
|
||||
@ -11,6 +11,7 @@ HTTP Response
|
||||
#include "common/io/io.h"
|
||||
#include "common/io/read.intern.h"
|
||||
#include "common/log.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/wait.h"
|
||||
|
||||
@ -77,7 +78,7 @@ httpResponseDone(HttpResponse *this)
|
||||
httpSessionFree(this->session);
|
||||
|
||||
// Only update the close stats after a successful response so it is not counted if there was an error/retry
|
||||
httpClientStat.close++;
|
||||
statInc(HTTP_STAT_CLOSE_STR);
|
||||
}
|
||||
// Else return it to the client so it can be reused
|
||||
else
|
||||
|
||||
@ -15,6 +15,7 @@ Socket Client
|
||||
#include "common/io/socket/common.h"
|
||||
#include "common/io/socket/session.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/wait.h"
|
||||
|
||||
@ -24,9 +25,11 @@ Io client type
|
||||
STRING_EXTERN(IO_CLIENT_SOCKET_TYPE_STR, IO_CLIENT_SOCKET_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
static SocketClientStat sckClientStatLocal;
|
||||
STRING_EXTERN(SOCKET_STAT_CLIENT_STR, SOCKET_STAT_CLIENT);
|
||||
STRING_EXTERN(SOCKET_STAT_RETRY_STR, SOCKET_STAT_RETRY);
|
||||
STRING_EXTERN(SOCKET_STAT_SESSION_STR, SOCKET_STAT_SESSION);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
@ -142,7 +145,7 @@ sckClientOpen(THIS_VOID)
|
||||
LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage());
|
||||
retry = true;
|
||||
|
||||
sckClientStatLocal.retry++;
|
||||
statInc(SOCKET_STAT_RETRY_STR);
|
||||
}
|
||||
else
|
||||
RETHROW();
|
||||
@ -151,7 +154,7 @@ sckClientOpen(THIS_VOID)
|
||||
}
|
||||
while (retry);
|
||||
|
||||
sckClientStatLocal.session++;
|
||||
statInc(SOCKET_STAT_SESSION_STR);
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
@ -208,7 +211,7 @@ sckClientNew(const String *host, unsigned int port, TimeMSec timeout)
|
||||
.timeout = timeout,
|
||||
};
|
||||
|
||||
sckClientStatLocal.object++;
|
||||
statInc(SOCKET_STAT_CLIENT_STR);
|
||||
|
||||
this = ioClientNew(driver, &sckClientInterface);
|
||||
}
|
||||
@ -216,21 +219,3 @@ sckClientNew(const String *host, unsigned int port, TimeMSec timeout)
|
||||
|
||||
FUNCTION_LOG_RETURN(IO_CLIENT, this);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
String *
|
||||
sckClientStatStr(void)
|
||||
{
|
||||
FUNCTION_TEST_VOID();
|
||||
|
||||
String *result = NULL;
|
||||
|
||||
if (sckClientStatLocal.object > 0)
|
||||
{
|
||||
result = strNewFmt(
|
||||
"socket statistics: objects %" PRIu64 ", sessions %" PRIu64 ", retries %" PRIu64, sckClientStatLocal.object,
|
||||
sckClientStatLocal.session, sckClientStatLocal.retry);
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RETURN(result);
|
||||
}
|
||||
|
||||
@ -16,24 +16,18 @@ Io client type
|
||||
STRING_DECLARE(IO_CLIENT_SOCKET_TYPE_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct SocketClientStat
|
||||
{
|
||||
uint64_t object; // Objects created
|
||||
uint64_t session; // Sessions created
|
||||
uint64_t retry; // Connection retries
|
||||
} SocketClientStat;
|
||||
#define SOCKET_STAT_CLIENT "socket.client" // Clients created
|
||||
STRING_DECLARE(SOCKET_STAT_CLIENT_STR);
|
||||
#define SOCKET_STAT_RETRY "socket.retry" // Connection retries
|
||||
STRING_DECLARE(SOCKET_STAT_RETRY_STR);
|
||||
#define SOCKET_STAT_SESSION "socket.session" // Sessions created
|
||||
STRING_DECLARE(SOCKET_STAT_SESSION_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
***********************************************************************************************************************************/
|
||||
IoClient *sckClientNew(const String *host, unsigned int port, TimeMSec timeout);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
// Statistics as a formatted string
|
||||
String *sckClientStatStr(void);
|
||||
|
||||
#endif
|
||||
|
||||
@ -16,6 +16,7 @@ TLS Client
|
||||
#include "common/io/tls/client.h"
|
||||
#include "common/io/tls/session.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/object.h"
|
||||
#include "common/wait.h"
|
||||
|
||||
@ -25,9 +26,11 @@ Io client type
|
||||
STRING_EXTERN(IO_CLIENT_TLS_TYPE_STR, IO_CLIENT_TLS_TYPE);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
static TlsClientStat tlsClientStatLocal;
|
||||
STRING_EXTERN(TLS_STAT_CLIENT_STR, TLS_STAT_CLIENT);
|
||||
STRING_EXTERN(TLS_STAT_RETRY_STR, TLS_STAT_RETRY);
|
||||
STRING_EXTERN(TLS_STAT_SESSION_STR, TLS_STAT_SESSION);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Object type
|
||||
@ -263,7 +266,7 @@ tlsClientOpen(THIS_VOID)
|
||||
LOG_DEBUG_FMT("retry %s: %s", errorTypeName(errorType()), errorMessage());
|
||||
retry = true;
|
||||
|
||||
tlsClientStatLocal.retry++;
|
||||
statInc(TLS_STAT_RETRY_STR);
|
||||
}
|
||||
else
|
||||
RETHROW();
|
||||
@ -276,7 +279,7 @@ tlsClientOpen(THIS_VOID)
|
||||
}
|
||||
MEM_CONTEXT_TEMP_END();
|
||||
|
||||
tlsClientStatLocal.session++;
|
||||
statInc(TLS_STAT_SESSION_STR);
|
||||
|
||||
// Verify that the certificate presented by the server is valid
|
||||
if (this->verifyPeer) // {vm_covered}
|
||||
@ -401,7 +404,7 @@ tlsClientNew(IoClient *ioClient, const String *host, TimeMSec timeout, bool veri
|
||||
}
|
||||
}
|
||||
|
||||
tlsClientStatLocal.object++;
|
||||
statInc(TLS_STAT_CLIENT_STR);
|
||||
|
||||
// Create client interface
|
||||
this = ioClientNew(driver, &tlsClientInterface);
|
||||
@ -410,21 +413,3 @@ tlsClientNew(IoClient *ioClient, const String *host, TimeMSec timeout, bool veri
|
||||
|
||||
FUNCTION_LOG_RETURN(IO_CLIENT, this);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
String *
|
||||
tlsClientStatStr(void)
|
||||
{
|
||||
FUNCTION_TEST_VOID();
|
||||
|
||||
String *result = NULL;
|
||||
|
||||
if (tlsClientStatLocal.object > 0)
|
||||
{
|
||||
result = strNewFmt(
|
||||
"tls statistics: objects %" PRIu64 ", sessions %" PRIu64 ", retries %" PRIu64, tlsClientStatLocal.object,
|
||||
tlsClientStatLocal.session, tlsClientStatLocal.retry);
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RETURN(result);
|
||||
}
|
||||
|
||||
@ -18,14 +18,14 @@ Io client type
|
||||
STRING_DECLARE(IO_CLIENT_TLS_TYPE_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics
|
||||
Statistics constants
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct TlsClientStat
|
||||
{
|
||||
uint64_t object; // Objects created
|
||||
uint64_t session; // Sessions created
|
||||
uint64_t retry; // Connection retries
|
||||
} TlsClientStat;
|
||||
#define TLS_STAT_CLIENT "tls.client" // Clients created
|
||||
STRING_DECLARE(TLS_STAT_CLIENT_STR);
|
||||
#define TLS_STAT_RETRY "tls.retry" // Connection retries
|
||||
STRING_DECLARE(TLS_STAT_RETRY_STR);
|
||||
#define TLS_STAT_SESSION "tls.session" // Sessions created
|
||||
STRING_DECLARE(TLS_STAT_SESSION_STR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Constructors
|
||||
|
||||
127
src/common/stat.c
Normal file
127
src/common/stat.c
Normal file
@ -0,0 +1,127 @@
|
||||
/***********************************************************************************************************************************
|
||||
Statistics Collector
|
||||
***********************************************************************************************************************************/
|
||||
#include "build.auto.h"
|
||||
|
||||
#include "common/debug.h"
|
||||
#include "common/memContext.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/type/list.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Stat output constants
|
||||
***********************************************************************************************************************************/
|
||||
VARIANT_STRDEF_EXTERN(STAT_VALUE_TOTAL_VAR, STAT_VALUE_TOTAL);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Cumulative statistics
|
||||
***********************************************************************************************************************************/
|
||||
typedef struct Stat
|
||||
{
|
||||
const String *key;
|
||||
uint64_t total;
|
||||
} Stat;
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Local data
|
||||
***********************************************************************************************************************************/
|
||||
struct
|
||||
{
|
||||
MemContext *memContext; // Mem context to store data in this struct
|
||||
List *stat; // Cumulative stats
|
||||
} statLocalData;
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
void
|
||||
statInit(void)
|
||||
{
|
||||
FUNCTION_TEST_VOID();
|
||||
|
||||
ASSERT(statLocalData.memContext == NULL);
|
||||
|
||||
MEM_CONTEXT_BEGIN(memContextTop())
|
||||
{
|
||||
MEM_CONTEXT_NEW_BEGIN("StatLocalData")
|
||||
{
|
||||
statLocalData.memContext = MEM_CONTEXT_NEW();
|
||||
statLocalData.stat = lstNewP(sizeof(Stat), .sortOrder = sortOrderAsc, .comparator = lstComparatorStr);
|
||||
}
|
||||
MEM_CONTEXT_NEW_END();
|
||||
}
|
||||
MEM_CONTEXT_END();
|
||||
|
||||
FUNCTION_TEST_RETURN_VOID();
|
||||
}
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Get the specified stat. If it doesn't already exist it will be created.
|
||||
***********************************************************************************************************************************/
|
||||
static Stat *
|
||||
statGetOrCreate(const String *key)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STRING, key);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(key != NULL);
|
||||
|
||||
// Attempt to find the stat
|
||||
Stat *stat = lstFind(statLocalData.stat, &key);
|
||||
|
||||
// If not found then create it
|
||||
if (stat == NULL)
|
||||
{
|
||||
// Add the new stat
|
||||
MEM_CONTEXT_BEGIN(lstMemContext(statLocalData.stat))
|
||||
{
|
||||
lstAdd(statLocalData.stat, &(Stat){.key = strDup(key)});
|
||||
}
|
||||
MEM_CONTEXT_END();
|
||||
|
||||
// Sort stats so this stat will be easier to find later
|
||||
lstSort(statLocalData.stat, sortOrderAsc);
|
||||
|
||||
// The stat might have moved so we'll need to find it and make sure we have the correct pointer
|
||||
stat = lstFind(statLocalData.stat, &key);
|
||||
ASSERT(stat != NULL);
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RETURN(stat);
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
void
|
||||
statInc(const String *key)
|
||||
{
|
||||
FUNCTION_TEST_BEGIN();
|
||||
FUNCTION_TEST_PARAM(STRING, key);
|
||||
FUNCTION_TEST_END();
|
||||
|
||||
ASSERT(statLocalData.memContext != NULL);
|
||||
ASSERT(key != NULL);
|
||||
|
||||
statGetOrCreate(key)->total++;
|
||||
|
||||
FUNCTION_TEST_RETURN();
|
||||
}
|
||||
|
||||
/**********************************************************************************************************************************/
|
||||
KeyValue *
|
||||
statToKv(void)
|
||||
{
|
||||
FUNCTION_TEST_VOID();
|
||||
|
||||
ASSERT(statLocalData.memContext != NULL);
|
||||
|
||||
KeyValue *result = kvNew();
|
||||
|
||||
for (unsigned int statIdx = 0; statIdx < lstSize(statLocalData.stat); statIdx++)
|
||||
{
|
||||
Stat *stat = lstGet(statLocalData.stat, statIdx);
|
||||
|
||||
KeyValue *statKv = kvPutKv(result, VARSTR(stat->key));
|
||||
kvAdd(statKv, STAT_VALUE_TOTAL_VAR, VARUINT64(stat->total));
|
||||
}
|
||||
|
||||
FUNCTION_TEST_RETURN(result);
|
||||
}
|
||||
35
src/common/stat.h
Normal file
35
src/common/stat.h
Normal file
@ -0,0 +1,35 @@
|
||||
/***********************************************************************************************************************************
|
||||
Statistics Collector
|
||||
|
||||
Collect simple statistics that can be output to a KeyValue for processing and logging. Each stat has a String that identifies it
|
||||
uniquely and will also be used in the output. Individual stats do not need to be created in advance since they will be created as
|
||||
needed at runtime. However, statInit() must be called before any other stat*() functions.
|
||||
|
||||
NOTE: Statistics are held in a sorted list so there is some cost involved in each lookup. In general, statistics should be used for
|
||||
relatively important or high-latency operations where measurements are critical. For instance, using statistics to count the
|
||||
iterations of a loop would likely be a bad idea.
|
||||
***********************************************************************************************************************************/
|
||||
#ifndef COMMON_STAT_H
|
||||
#define COMMON_STAT_H
|
||||
|
||||
#include "common/type/variant.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Statistics output constants
|
||||
***********************************************************************************************************************************/
|
||||
#define STAT_VALUE_TOTAL "total"
|
||||
VARIANT_DECLARE(STAT_VALUE_TOTAL_VAR);
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Functions
|
||||
***********************************************************************************************************************************/
|
||||
// Initialize the stats collector
|
||||
void statInit(void);
|
||||
|
||||
// Increment stat by one
|
||||
void statInc(const String *key);
|
||||
|
||||
// Output stats to a KeyValue
|
||||
KeyValue *statToKv(void);
|
||||
|
||||
#endif
|
||||
@ -31,6 +31,7 @@ Main
|
||||
#include "common/debug.h"
|
||||
#include "common/error.h"
|
||||
#include "common/exit.h"
|
||||
#include "common/stat.h"
|
||||
#include "config/config.h"
|
||||
#include "config/load.h"
|
||||
#include "postgres/interface.h"
|
||||
@ -52,6 +53,9 @@ main(int argListSize, const char *argList[])
|
||||
// Initialize command with the start time
|
||||
cmdInit();
|
||||
|
||||
// Initialize statistics collector
|
||||
statInit();
|
||||
|
||||
volatile bool result = 0;
|
||||
volatile bool error = false;
|
||||
|
||||
|
||||
@ -73,7 +73,7 @@ unit:
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: mem-context
|
||||
total: 7
|
||||
define-test: -DNO_MEM_CONTEXT -DNO_LOG
|
||||
define-test: -DNO_MEM_CONTEXT -DNO_LOG -DNO_STAT
|
||||
|
||||
coverage:
|
||||
common/memContext: full
|
||||
@ -212,6 +212,13 @@ unit:
|
||||
coverage:
|
||||
common/type/xml: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: stat
|
||||
total: 1
|
||||
|
||||
coverage:
|
||||
common/stat: full
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: user
|
||||
total: 1
|
||||
@ -774,7 +781,7 @@ performance:
|
||||
test:
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: type
|
||||
total: 3
|
||||
total: 4
|
||||
|
||||
# ----------------------------------------------------------------------------------------------------------------------------
|
||||
- name: storage
|
||||
|
||||
@ -2429,11 +2429,9 @@ P00 INFO: last backup label = [BACKUP-FULL-3], version = [VERSION-1]
|
||||
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base-2/base/base/base2.txt (9B, 100%) checksum cafac3c59553f2cfde41ce2e62e7662295f108c0
|
||||
P00 INFO: diff backup size = 9B
|
||||
P00 INFO: new backup label = [BACKUP-DIFF-5]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/backup/pgbackrest.conf --lock-path=[TEST_PATH]/backup/lock --log-level-console=info --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 INFO: option 'repo1-retention-archive' is not set - archive logs will not be expired
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 INFO: expire command end: completed successfully
|
||||
|
||||
+ supplemental file: [TEST_PATH]/db-primary/pgbackrest.conf
|
||||
|
||||
@ -29,9 +29,7 @@ stanza-create db - stanza create (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-path=/ --repo1-s3-bucket=pgbackrest-dev --repo1-s3-endpoint=s3.amazonaws.com --repo1-s3-key=<redacted> --repo1-s3-key-secret=<redacted> --repo1-s3-region=us-east-1 --no-repo1-s3-verify-tls --repo1-type=s3 --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
|
||||
@ -5,9 +5,7 @@ stanza-create db - create required data for stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
|
||||
@ -6,9 +6,7 @@ stanza-create db - fail on missing control file (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 ERROR: [055]: raised from remote-0 protocol on 'db-primary': unable to open missing file '[TEST_PATH]/db-primary/db/base/global/pg_control' for read
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: aborted with exception [055]
|
||||
|
||||
stanza-upgrade db - fail on stanza not initialized since archive.info is missing (backup host)
|
||||
@ -22,18 +20,14 @@ P00 ERROR: [055]: unable to load info file '/archive/db/archive.info' or '/arch
|
||||
HINT: is archive_command configured correctly in postgresql.conf?
|
||||
HINT: has a stanza-create been performed?
|
||||
HINT: use --no-archive-check to disable archive checks during backup if you have an alternate archiving scheme.
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-upgrade command end: aborted with exception [055]
|
||||
|
||||
stanza-create db - successfully create the stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-create
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -83,9 +77,7 @@ stanza-create db - do not fail on rerun of stanza-create - info files exist and
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-create command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 INFO: stanza 'db' already exists and is valid
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -138,9 +130,7 @@ P00 WARN: option --force is no longer supported
|
||||
P00 ERROR: [028]: backup and archive info files exist but do not match the database
|
||||
HINT: is this the correct stanza?
|
||||
HINT: did an error occur during stanza-upgrade?
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-create command end: aborted with exception [028]
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -190,9 +180,7 @@ stanza-upgrade db - already up to date (backup host)
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 INFO: stanza 'db' is already up to date
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -258,9 +246,7 @@ stanza-upgrade db - successful upgrade creates additional history (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -328,14 +314,10 @@ P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archiv
|
||||
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
|
||||
P00 INFO: full backup size = 48MB
|
||||
P00 INFO: new backup label = [BACKUP-FULL-1]
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/backup/pgbackrest.conf --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: expire command end: completed successfully
|
||||
|
||||
+ supplemental file: [TEST_PATH]/db-primary/pgbackrest.conf
|
||||
@ -404,9 +386,7 @@ stanza-upgrade db - successfully upgrade (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db --no-online stanza-upgrade
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-upgrade command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --no-online --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-upgrade command end: completed successfully
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -466,9 +446,7 @@ P00 ERROR: [028]: backup info file and archive info file do not match
|
||||
archive: id = 2, version = 9.5, system-id = 1000000000000000095
|
||||
backup : id = 3, version = 9.5, system-id = 1000000000000000095
|
||||
HINT: this may be a symptom of repository corruption!
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-upgrade command end: aborted with exception [028]
|
||||
|
||||
+ supplemental file: /backup/db/backup.info
|
||||
@ -534,15 +512,11 @@ P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archiv
|
||||
P01 INFO: backup file db-primary:[TEST_PATH]/db-primary/db/base/pg_xlog/archive_status/000000010000000100000001.ready (0B, 100%)
|
||||
P00 INFO: full backup size = 48MB
|
||||
P00 INFO: new backup label = [BACKUP-FULL-2]
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: backup command end: completed successfully
|
||||
P00 INFO: expire command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --config=[TEST_PATH]/backup/pgbackrest.conf --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-retention-full=2 --repo1-type=azure --stanza=db
|
||||
P00 INFO: remove archive path: /archive/db/9.3-1
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: expire command end: completed successfully
|
||||
|
||||
+ supplemental file: [TEST_PATH]/db-primary/pgbackrest.conf
|
||||
@ -613,9 +587,7 @@ stanza-delete db - fail on missing stop file (backup host)
|
||||
P00 INFO: stanza-delete command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 ERROR: [055]: stop file does not exist for stanza 'db'
|
||||
HINT: has the pgbackrest stop command been run on this server for this stanza?
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-delete command end: aborted with exception [055]
|
||||
|
||||
db must not exist for successful delete
|
||||
@ -638,9 +610,7 @@ stanza-delete db - successfully delete the stanza (backup host)
|
||||
> [CONTAINER-EXEC] backup [BACKREST-BIN] --config=[TEST_PATH]/backup/pgbackrest.conf --stanza=db stanza-delete
|
||||
------------------------------------------------------------------------------------------------------------------------------------
|
||||
P00 INFO: stanza-delete command begin [BACKREST-VERSION]: --buffer-size=[BUFFER-SIZE] --compress-level-network=1 --config=[TEST_PATH]/backup/pgbackrest.conf --db-timeout=45 --lock-path=[TEST_PATH]/backup/lock --log-level-console=detail --log-level-file=[LOG-LEVEL-FILE] --log-level-stderr=off --log-path=[TEST_PATH]/backup/log[] --no-log-timestamp --pg1-host=db-primary --pg1-host-cmd=[BACKREST-BIN] --pg1-host-config=[TEST_PATH]/db-primary/pgbackrest.conf --pg1-host-user=[USER-1] --pg1-path=[TEST_PATH]/db-primary/db/base --protocol-timeout=60 --repo1-azure-account=<redacted> --repo1-azure-container=azContainer --repo1-azure-host=azure --repo1-azure-key=<redacted> --no-repo1-azure-verify-tls --repo1-cipher-pass=<redacted> --repo1-cipher-type=aes-256-cbc --repo1-path=/ --repo1-type=azure --stanza=db
|
||||
P00 DETAIL: socket statistics:[SOCKET-STATISTICS]
|
||||
P00 DETAIL: tls statistics:[TLS-STATISTICS]
|
||||
P00 INFO: http statistics:[HTTP-STATISTICS]
|
||||
P00 DETAIL: statistics: STATISTICS
|
||||
P00 INFO: stanza-delete command end: completed successfully
|
||||
|
||||
db must not exist for successful delete
|
||||
|
||||
@ -390,9 +390,10 @@ sub regExpReplaceAll
|
||||
$strLine = $self->regExpReplace(
|
||||
$strLine, 'BACKUP-EXPR', 'strExpression \= \_[0-9]{8}\-[0-9]{6}', '\_[0-9]{8}\-[0-9]{6}$', false);
|
||||
|
||||
$strLine = $self->regExpReplace($strLine, 'SOCKET-STATISTICS', 'socket statistics\:.*$', '[^\:]+$', false);
|
||||
$strLine = $self->regExpReplace($strLine, 'TLS-STATISTICS', 'tls statistics\:.*$', '[^\:]+$', false);
|
||||
$strLine = $self->regExpReplace($strLine, 'HTTP-STATISTICS', 'http statistics\:.*$', '[^\:]+$', false);
|
||||
if ($strLine =~ /^P00 DETAIL\: statistics\: /)
|
||||
{
|
||||
$strLine = 'P00 DETAIL: statistics: STATISTICS'
|
||||
}
|
||||
|
||||
$strLine = $self->regExpReplace($strLine, 'GROUP', 'strGroup = [^ \n,\[\]]+', '[^ \n,\[\]]+$');
|
||||
$strLine = $self->regExpReplace($strLine, 'GROUP', 'unknown group in backup manifest mapped to \'[^\']+', '[^\']+$');
|
||||
|
||||
@ -4,7 +4,7 @@ Test Common Command Routines
|
||||
#include <fcntl.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "storage/storage.h"
|
||||
#include "common/stat.h"
|
||||
#include "version.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
@ -121,16 +121,15 @@ testRun(void)
|
||||
|
||||
cfgOptionSet(cfgOptLogTimestamp, cfgSourceParam, varNewBool(true));
|
||||
|
||||
httpClientNew(tlsClientNew(sckClientNew(STRDEF("BOGUS"), 443, 1000), STRDEF("BOGUS"), 1000, true, NULL, NULL), 1000);
|
||||
// Add one stat to make sure it gets output
|
||||
statInc(STRDEF("test"));
|
||||
|
||||
harnessLogLevelSet(logLevelDetail);
|
||||
|
||||
TEST_RESULT_VOID(cmdEnd(0, NULL), "command end with success");
|
||||
hrnLogReplaceAdd("\\([0-9]+ms\\)", "[0-9]+", "TIME", false);
|
||||
TEST_RESULT_LOG(
|
||||
"P00 DETAIL: socket statistics: objects 1, sessions 0, retries 0\n"
|
||||
"P00 DETAIL: tls statistics: objects 1, sessions 0, retries 0\n"
|
||||
"P00 INFO: http statistics: objects 1, sessions 0, requests 0, retries 0, closes 0\n"
|
||||
"P00 DETAIL: statistics: {\"test\":{\"total\":1}}\n"
|
||||
"P00 INFO: archive-get command end: completed successfully ([TIME]ms)");
|
||||
|
||||
harnessLogLevelReset();
|
||||
|
||||
@ -178,11 +178,6 @@ testRun(void)
|
||||
{
|
||||
HttpClient *client = NULL;
|
||||
|
||||
// Reset statistics
|
||||
httpClientStat = (HttpClientStat){0};
|
||||
|
||||
TEST_RESULT_STR(httpClientStatStr(), NULL, "no stats yet");
|
||||
|
||||
TEST_ASSIGN(client, httpClientNew(sckClientNew(strNew("localhost"), hrnServerPort(0), 500), 500), "new client");
|
||||
|
||||
TEST_ERROR_FMT(
|
||||
@ -630,7 +625,7 @@ testRun(void)
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("statistics exist");
|
||||
|
||||
TEST_RESULT_BOOL(httpClientStatStr() != NULL, true, "check");
|
||||
TEST_RESULT_BOOL(varLstSize(kvKeyList(statToKv())) > 0, true, "check");
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
|
||||
@ -366,12 +366,6 @@ testRun(void)
|
||||
IoClient *client = NULL;
|
||||
IoSession *session = NULL;
|
||||
|
||||
// Reset statistics
|
||||
sckClientStatLocal = (SocketClientStat){0};
|
||||
TEST_RESULT_STR(sckClientStatStr(), NULL, "no stats yet");
|
||||
tlsClientStatLocal = (TlsClientStat){0};
|
||||
TEST_RESULT_STR(tlsClientStatStr(), NULL, "no stats yet");
|
||||
|
||||
HARNESS_FORK_BEGIN()
|
||||
{
|
||||
HARNESS_FORK_CHILD_BEGIN(0, true)
|
||||
@ -509,8 +503,7 @@ testRun(void)
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("stastistics exist");
|
||||
|
||||
TEST_RESULT_BOOL(sckClientStatStr() != NULL, true, "check socket");
|
||||
TEST_RESULT_BOOL(tlsClientStatStr() != NULL, true, "check tls");
|
||||
TEST_RESULT_BOOL(varLstSize(kvKeyList(statToKv())) > 0, true, "check");
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
|
||||
33
test/src/module/common/statTest.c
Normal file
33
test/src/module/common/statTest.c
Normal file
@ -0,0 +1,33 @@
|
||||
/***********************************************************************************************************************************
|
||||
Test Statistics Collector
|
||||
***********************************************************************************************************************************/
|
||||
#include "common/type/json.h"
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
Test Run
|
||||
***********************************************************************************************************************************/
|
||||
void
|
||||
testRun(void)
|
||||
{
|
||||
FUNCTION_HARNESS_VOID();
|
||||
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("all"))
|
||||
{
|
||||
const String *statTlsClient = STRDEF("tls.client");
|
||||
const String *statHttpSession = STRDEF("http.session");
|
||||
|
||||
TEST_RESULT_UINT(lstSize(statLocalData.stat), 0, "stat list is empty");
|
||||
|
||||
TEST_RESULT_VOID(statInc(statTlsClient), "inc tls.client");
|
||||
TEST_RESULT_UINT(lstSize(statLocalData.stat), 1, "stat list has one stat");
|
||||
TEST_RESULT_VOID(statInc(statTlsClient), "inc tls.client");
|
||||
TEST_RESULT_UINT(lstSize(statLocalData.stat), 1, "stat list has one stat");
|
||||
TEST_RESULT_VOID(statInc(statHttpSession), "inc http.session");
|
||||
TEST_RESULT_UINT(lstSize(statLocalData.stat), 2, "stat list has two stats");
|
||||
|
||||
TEST_RESULT_STR_Z(jsonFromKv(statToKv()), "{\"http.session\":{\"total\":1},\"tls.client\":{\"total\":2}}", "stat output");
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
}
|
||||
@ -11,6 +11,7 @@ running out of memory on the test systems or taking an undue amount of time. It
|
||||
#include "common/ini.h"
|
||||
#include "common/io/bufferRead.h"
|
||||
#include "common/io/bufferWrite.h"
|
||||
#include "common/stat.h"
|
||||
#include "common/time.h"
|
||||
#include "common/type/list.h"
|
||||
#include "common/type/object.h"
|
||||
@ -291,5 +292,46 @@ testRun(void)
|
||||
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
||||
}
|
||||
|
||||
// Make sure statistics collector performs well
|
||||
// *****************************************************************************************************************************
|
||||
if (testBegin("statistics collector"))
|
||||
{
|
||||
CHECK(testScale() <= 1000000);
|
||||
|
||||
// Setup a list of stats to use for testing
|
||||
#define TEST_STAT_TOTAL 100
|
||||
String *statList[TEST_STAT_TOTAL];
|
||||
|
||||
for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
|
||||
statList[statIdx] = strNewFmt("STAT%u", statIdx);
|
||||
|
||||
uint64_t runTotal = (uint64_t)testScale() * (uint64_t)100000;
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE_FMT("update %d stats %" PRIu64 " times", TEST_STAT_TOTAL, runTotal);
|
||||
|
||||
TimeMSec timeBegin = timeMSec();
|
||||
|
||||
for (uint64_t runIdx = 0; runIdx < runTotal; runIdx++)
|
||||
{
|
||||
for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
|
||||
statInc(statList[statIdx]);
|
||||
}
|
||||
|
||||
TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
|
||||
|
||||
// -------------------------------------------------------------------------------------------------------------------------
|
||||
TEST_TITLE("check stats have correct values");
|
||||
|
||||
KeyValue *statKv = statToKv();
|
||||
|
||||
for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
|
||||
{
|
||||
TEST_RESULT_UINT(
|
||||
varUInt64(kvGet(varKv(kvGet(statKv, VARSTR(statList[statIdx]))), STAT_VALUE_TOTAL_VAR)), runTotal, "check stat %u",
|
||||
statIdx);
|
||||
}
|
||||
}
|
||||
|
||||
FUNCTION_HARNESS_RESULT_VOID();
|
||||
}
|
||||
|
||||
@ -48,6 +48,10 @@ Includes that are not generally used by tests
|
||||
|
||||
#include "common/io/socket/common.h"
|
||||
|
||||
#ifndef NO_STAT
|
||||
#include "common/stat.h"
|
||||
#endif
|
||||
|
||||
/***********************************************************************************************************************************
|
||||
main - run the tests
|
||||
***********************************************************************************************************************************/
|
||||
@ -72,6 +76,11 @@ main(int argListSize, const char *argList[])
|
||||
|
||||
int result = 0;
|
||||
|
||||
// Initialize statistics
|
||||
#ifndef NO_STAT
|
||||
statInit();
|
||||
#endif
|
||||
|
||||
// Use aggressive keep-alive settings for testing
|
||||
sckInit(false, true, 2, 5, 5);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user