db/pgbackrest
1
0
Fork 0
mirror of https://github.com/pgbackrest/pgbackrest.git synced 2025-03-03 14:52:21 +02:00
Code Issues Releases Activity

Aggregate error retries in ErrorRetry output.

Browse Source 
If there are a lot of retries then the output might be very large and even be truncated by the error module. Either way, it is not good information for the user.

When a message is repeated, aggregate so that total retries and time range are output for the message. This provides helpful information about what happened without overwhelming the user with data.
This commit is contained in:
David Steele 2023-09-09 12:54:55 -04:00
parent 5314dbffc7
commit c1805134b3
5 changed files with 92 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/common/error/retry.c
View File

@ -3,9 +3,12 @@ Error Retry Message
***********************************************************************************************************************************/
#include "build.auto.h"
#include "string.h"
#include "common/debug.h"
#include "common/error/retry.h"
#include "common/time.h"
#include "common/type/list.h"
#include "common/type/string.h"
/***********************************************************************************************************************************
@ -14,10 +17,21 @@ Object type
struct ErrorRetry
{
ErrorRetryPub pub; // Publicly accessible variables
String *messageLast; // Last error message
const String *message; // First error message
List *list; // List of retry errors
TimeMSec timeBegin; // Time error retries began
};
// Keep track of retries with the same error
typedef struct ErrorRetryItem
{
String *message; // Error message
unsigned int total; // Total errors with this message
const ErrorType *type; // Error type
TimeMSec retryFirst; // First retry
TimeMSec retryLast; // Last Retry
} ErrorRetryItem;
/**********************************************************************************************************************************/
FN_EXTERN ErrorRetry *
errRetryNew(void)
@ -29,6 +43,7 @@ errRetryNew(void)
*this = (ErrorRetry)
{
.timeBegin = timeMSec(),
.list = lstNewP(sizeof(ErrorRetryItem), .comparator = lstComparatorStr),
};
}
OBJ_NEW_END();
@ -36,6 +51,37 @@ errRetryNew(void)
FUNCTION_TEST_RETURN(ERROR_RETRY, this);
}
/**********************************************************************************************************************************/
FN_EXTERN String *
errRetryMessage(const ErrorRetry *const this)
{
FUNCTION_TEST_BEGIN();
FUNCTION_TEST_PARAM(ERROR_RETRY, this);
FUNCTION_TEST_END();
ASSERT(this->message != NULL);
// Always include first message
String *const result = strCat(strNew(), this->message);
// Add retries, varying the message depending on how many retries there were
for (unsigned int listIdx = 0; listIdx < lstSize(this->list); listIdx++)
{
const ErrorRetryItem *const error = lstGet(this->list, listIdx);
strCatFmt(result, "\n [%s] ", errorTypeName(error->type));
if (error->retryFirst == error->retryLast)
strCatFmt(result, "on retry at %" PRIu64, error->retryFirst);
else
strCatFmt(result, "on %u retries from %" PRIu64 "-%" PRIu64, error->total, error->retryFirst, error->retryLast);
strCatFmt(result, "ms: %s", strZ(error->message));
}
FUNCTION_TEST_RETURN(STRING, result);
}
/**********************************************************************************************************************************/
FN_EXTERN void
errRetryAdd(ErrorRetry *const this)
@ -44,37 +90,45 @@ errRetryAdd(ErrorRetry *const this)
FUNCTION_TEST_PARAM(ERROR_RETRY, this);
FUNCTION_TEST_END();
// If the first error
if (this->messageLast == NULL)
// Set type on first error
if (this->pub.type == NULL)
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.type = errorType();
this->pub.message = strCatZ(strNew(), errorMessage());
this->messageLast = strCatZ(strNew(), errorMessage());
this->message = strNewZ(errorMessage());
}
MEM_CONTEXT_OBJ_END();
}
// Else on each retry
else
{
strCatFmt(
this->pub.message, "\n[%s] on retry after %" PRIu64 "ms: ", errorTypeName(errorType()),
timeMSec() - this->timeBegin);
// If error is not found then add it
const String *const message = STR(errorMessage());
const TimeMSec retryTime = timeMSec() - this->timeBegin;
ErrorRetryItem *const error = lstFind(this->list, &message);
// If the message is the same as the last message
if (strEqZ(this->messageLast, errorMessage()))
if (error == NULL)
{
strCatZ(this->pub.message, "[same message]");
MEM_CONTEXT_OBJ_BEGIN(this->list)
{
const ErrorRetryItem errorNew =
{
.type = errorType(),
.total = 1,
.message = strDup(message),
.retryFirst = retryTime,
.retryLast = retryTime,
};
lstAdd(this->list, &errorNew);
}
MEM_CONTEXT_OBJ_END();
}
// Else append new message
// Else update the error found
else
{
strCatZ(this->pub.message, errorMessage());
// Update last message
strTrunc(this->messageLast);
strCatZ(this->messageLast, errorMessage());
error->total++;
error->retryLast = retryTime;
}
}

10
src/common/error/retry.h
View File

@ -2,7 +2,8 @@
Error Retry Message
Accumulate errors during retries to provide a single, coherent error message to the user that includes information about the
original error and all the failed retries.
original error and all the failed retries. Retries with the same error message are aggregated to keep the error message as short as
possible.
***********************************************************************************************************************************/
#ifndef COMMON_ERROR_RETRY_H
#define COMMON_ERROR_RETRY_H
@ -33,7 +34,6 @@ Getters/Setters
typedef struct ErrorRetryPub
{
const ErrorType *type; // Error type
String *message; // Error message
} ErrorRetryPub;
// Get error type
@ -44,11 +44,7 @@ errRetryType(const ErrorRetry *const this)
}
// Get error message
FN_INLINE_ALWAYS const String *
errRetryMessage(const ErrorRetry *const this)
{
return THIS_PUB(ErrorRetry)->message;
}
FN_EXTERN String *errRetryMessage(const ErrorRetry *this);
/***********************************************************************************************************************************
Destructor

2
test/define.yaml
View File

@ -179,7 +179,7 @@ unit:
shim:
common/error/retry:
function:
- errRetryAdd
- errRetryMessage
coverage:
- common/error/retry

28
test/src/common/harnessErrorRetry.c
View File

@ -17,32 +17,28 @@ static struct
} hrnErrorRetryLocal;
/**********************************************************************************************************************************/
void
errRetryAdd(ErrorRetry *const this)
String *
errRetryMessage(const ErrorRetry *const this)
{
FUNCTION_HARNESS_BEGIN();
FUNCTION_HARNESS_PARAM(ERROR_RETRY, this);
FUNCTION_HARNESS_END();
ASSERT(this->message != NULL);
String *result = NULL;
if (!hrnErrorRetryLocal.detailEnable)
{
if (this->messageLast == NULL)
{
MEM_CONTEXT_OBJ_BEGIN(this)
{
this->pub.type = errorType();
this->pub.message = strCatZ(strNew(), errorMessage());
this->messageLast = strNewZ(errorMessage());
}
MEM_CONTEXT_OBJ_END();
}
else if (strEq(this->pub.message, this->messageLast))
strCatFmt(this->pub.message, "\n[RETRY DETAIL OMITTED]");
result = strCat(strNew(), this->message);
if (lstSize(this->list) > 0)
strCatZ(result, "\n[RETRY DETAIL OMITTED]");
}
else
errRetryAdd_SHIMMED(this);
result = errRetryMessage_SHIMMED(this);
FUNCTION_HARNESS_RETURN_VOID();
FUNCTION_HARNESS_RETURN(STRING, result);
}
/**********************************************************************************************************************************/

9
test/src/module/common/errorRetryTest.c
View File

@ -31,7 +31,7 @@ testRun(void)
TRY_BEGIN()
{
THROW(KernelError, "message1");
THROW(KernelError, "message2");
}
CATCH_ANY()
{
@ -89,7 +89,7 @@ testRun(void)
TRY_BEGIN()
{
THROW(ServiceError, "message2");
THROW(ServiceError, "message1");
}
CATCH_ANY()
{
@ -101,9 +101,8 @@ testRun(void)
TEST_RESULT_STR_Z(
errRetryMessage(retry),
"message1\n"
"[FormatError] on retry after 50ms: [same message]\n"
"[KernelError] on retry after 75ms: message2\n"
"[ServiceError] on retry after 150ms: [same message]",
" [FormatError] on 2 retries from 50-150ms: message1\n"
" [KernelError] on retry at 75ms: message2",
"error message");
}
}