From a72cc49e8a08dd61cbcf61cac664601c70bc9517 Mon Sep 17 00:00:00 2001 From: winlin Date: Tue, 10 Jun 2025 14:53:08 -0400 Subject: [PATCH] avformat/tls_openssl: fix X509 cert serial number might be negative See RFC5280 4.1.2.2 Signed-off-by: Jack Lau --- libavformat/tls_openssl.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavformat/tls_openssl.c b/libavformat/tls_openssl.c index 65b2966688..bab2e711c6 100644 --- a/libavformat/tls_openssl.c +++ b/libavformat/tls_openssl.c @@ -315,7 +315,8 @@ end: static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerprint) { - int ret = 0, serial, expire_day; + int ret = 0, expire_day; + uint64_t serial; const char *aor = "lavf"; X509_NAME* subject = NULL; @@ -330,8 +331,8 @@ static int openssl_gen_certificate(EVP_PKEY *pkey, X509 **cert, char **fingerpri goto enomem_end; } - serial = (int)av_get_random_seed(); - if (ASN1_INTEGER_set(X509_get_serialNumber(*cert), serial) != 1) { + serial = av_get_random_seed(); + if (ASN1_INTEGER_set_uint64(X509_get_serialNumber(*cert), serial) != 1) { av_log(NULL, AV_LOG_ERROR, "TLS: Failed to set serial, %s\n", ERR_error_string(ERR_get_error(), NULL)); goto einval_end; }