libavformat: LibreSSL (libtls) support

Signed-off-by: sfan5 <sfan5@live.de>

Changelog

@@ -26,6 +26,7 @@ version <next>:
 - video fillborders filter
 - video setrange filter
 - nsp demuxer
+- support LibreSSL (via libtls)
 
 
 version 3.4:

configure

@@ -214,7 +214,7 @@ External library support:
   --enable-gmp             enable gmp, needed for rtmp(t)e support
                            if openssl or librtmp is not used [no]
   --enable-gnutls          enable gnutls, needed for https support
-                           if openssl is not used [no]
+                           if openssl or libtls is not used [no]
   --disable-iconv          disable iconv [autodetect]
   --enable-jni             enable JNI support [no]
   --enable-ladspa          enable LADSPA audio filtering [no]
@@ -259,6 +259,8 @@ External library support:
   --enable-libssh          enable SFTP protocol via libssh [no]
   --enable-libtesseract    enable Tesseract, needed for ocr filter [no]
   --enable-libtheora       enable Theora encoding via libtheora [no]
+  --enable-libtls          enable LibreSSL (via libtls), needed for https support
+                           if openssl or gnutls is not used [no]
   --enable-libtwolame      enable MP2 encoding via libtwolame [no]
   --enable-libv4l2         enable libv4l2/v4l-utils [no]
   --enable-libvidstab      enable video stabilization using vid.stab [no]
@@ -292,7 +294,7 @@ External library support:
   --enable-opencl          enable OpenCL processing [no]
   --enable-opengl          enable OpenGL rendering [no]
   --enable-openssl         enable openssl, needed for https support
-                           if gnutls is not used [no]
+                           if gnutls or libtls is not used [no]
   --disable-sndio          disable sndio support [autodetect]
   --disable-schannel       disable SChannel SSP, needed for TLS support on
                            Windows if openssl and gnutls are not used [autodetect]
@@ -1563,6 +1565,7 @@ EXTERNAL_LIBRARY_NONFREE_LIST="
     libndi_newtek
     libfdk_aac
     openssl
+    libtls
 "
 
 EXTERNAL_LIBRARY_VERSION3_LIST="
@@ -3143,6 +3146,7 @@ librtmpt_protocol_deps="librtmp"
 librtmpte_protocol_deps="librtmp"
 libsmbclient_protocol_deps="libsmbclient gplv3"
 libssh_protocol_deps="libssh"
+libtls_conflict="openssl gnutls"
 mmsh_protocol_select="http_protocol"
 mmst_protocol_select="network"
 rtmp_protocol_conflict="librtmp_protocol"
@@ -3160,13 +3164,13 @@ rtmpte_protocol_suggest="zlib"
 rtmpts_protocol_select="ffrtmphttp_protocol https_protocol"
 rtmpts_protocol_suggest="zlib"
 rtp_protocol_select="udp_protocol"
-schannel_conflict="openssl gnutls"
+schannel_conflict="openssl gnutls libtls"
 sctp_protocol_deps="struct_sctp_event_subscribe struct_msghdr_msg_flags"
 sctp_protocol_select="network"
-securetransport_conflict="openssl gnutls"
+securetransport_conflict="openssl gnutls libtls"
 srtp_protocol_select="rtp_protocol srtp"
 tcp_protocol_select="network"
-tls_protocol_deps_any="gnutls openssl schannel securetransport"
+tls_protocol_deps_any="gnutls openssl schannel securetransport libtls"
 tls_protocol_select="tcp_protocol"
 udp_protocol_select="network"
 udplite_protocol_select="network"
@@ -5890,6 +5894,8 @@ enabled libssh            && require_pkg_config libssh libssh libssh/sftp.h sftp
 enabled libspeex          && require_pkg_config libspeex speex speex/speex.h speex_decoder_init
 enabled libtesseract      && require_pkg_config libtesseract tesseract tesseract/capi.h TessBaseAPICreate
 enabled libtheora         && require libtheora theora/theoraenc.h th_info_init -ltheoraenc -ltheoradec -logg
+enabled libtls            && { use_pkg_config libtls libtls tls.h tls_configure ||
+                               require libtls tls.h tls_configure -ltls; }
 enabled libtwolame        && require libtwolame twolame.h twolame_init -ltwolame &&
                              { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame ||
                                die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }

doc/protocols.texi

@@ -1286,7 +1286,7 @@ If enabled, try to verify the peer that we are communicating with.
 Note, if using OpenSSL, this currently only makes sure that the
 peer certificate is signed by one of the root certificates in the CA
 database, but it does not validate that the certificate actually
-matches the host name we are trying to connect to. (With GnuTLS,
+matches the host name we are trying to connect to. (With other backends,
 the host name is validated as well.)
 
 This is disabled by default since it requires a CA database to be

libavformat/Makefile

@@ -593,6 +593,7 @@ OBJS-$(CONFIG_SUBFILE_PROTOCOL)          += subfile.o
 OBJS-$(CONFIG_TEE_PROTOCOL)              += teeproto.o tee_common.o
 OBJS-$(CONFIG_TCP_PROTOCOL)              += tcp.o
 TLS-OBJS-$(CONFIG_GNUTLS)                += tls_gnutls.o
+TLS-OBJS-$(CONFIG_LIBTLS)                += tls_libtls.o
 TLS-OBJS-$(CONFIG_OPENSSL)               += tls_openssl.o
 TLS-OBJS-$(CONFIG_SECURETRANSPORT)       += tls_securetransport.o
 TLS-OBJS-$(CONFIG_SCHANNEL)              += tls_schannel.o

libavformat/tls_libtls.c

@@ -0,0 +1,207 @@
+/*
+ * TLS/SSL Protocol
+ * Copyright (c) 2011 Martin Storsjo
+ * Copyright (c) 2017 sfan5 <sfan5@live.de>
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "avformat.h"
+#include "internal.h"
+#include "network.h"
+#include "url.h"
+#include "tls.h"
+#include "libavcodec/internal.h"
+#include "libavutil/avutil.h"
+#include "libavutil/opt.h"
+
+#include <tls.h>
+
+typedef struct TLSContext {
+    const AVClass *class;
+    TLSShared tls_shared;
+    struct tls *ctx;
+} TLSContext;
+
+static int ff_tls_close(URLContext *h)
+{
+    TLSContext *p = h->priv_data;
+    if (p->ctx) {
+        tls_close(p->ctx);
+        tls_free(p->ctx);
+    }
+    if (p->tls_shared.tcp)
+        ffurl_close(p->tls_shared.tcp);
+    return 0;
+}
+
+static ssize_t tls_read_callback(struct tls *ctx, void *buf, size_t buflen, void *cb_arg)
+{
+    URLContext *h = (URLContext*) cb_arg;
+    int ret = ffurl_read(h, buf, buflen);
+    if (ret == AVERROR(EAGAIN))
+        return TLS_WANT_POLLIN;
+    else if (ret == AVERROR_EXIT)
+        return 0;
+    return ret >= 0 ? ret : -1;
+}
+
+static ssize_t tls_write_callback(struct tls *ctx, const void *buf, size_t buflen, void *cb_arg)
+{
+    URLContext *h = (URLContext*) cb_arg;
+    int ret = ffurl_write(h, buf, buflen);
+    if (ret == AVERROR(EAGAIN))
+        return TLS_WANT_POLLOUT;
+    else if (ret == AVERROR_EXIT)
+        return 0;
+    return ret >= 0 ? ret : -1;
+}
+
+static int ff_tls_open(URLContext *h, const char *uri, int flags, AVDictionary **options)
+{
+    TLSContext *p = h->priv_data;
+    TLSShared *c = &p->tls_shared;
+    struct tls_config *cfg = NULL;
+    int ret;
+
+    if (tls_init() == -1) {
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    if ((ret = ff_tls_open_underlying(c, h, uri, options)) < 0)
+        goto fail;
+
+    p->ctx = !c->listen ? tls_client() : tls_server();
+    if (!p->ctx) {
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+
+    cfg = tls_config_new();
+    if (!p->ctx) {
+        ret = AVERROR(EIO);
+        goto fail;
+    }
+    if (tls_config_set_protocols(cfg, TLS_PROTOCOLS_ALL) == -1)
+        goto err_config;
+    // While TLSv1.0 and TLSv1.1 are already enabled by the above,
+    // we need to be less strict with ciphers so it works in practice.
+    if (tls_config_set_ciphers(cfg, "compat") == -1)
+        goto err_config;
+    if (c->ca_file && tls_config_set_ca_file(cfg, c->ca_file) == -1)
+        goto err_config;
+    if (c->cert_file && tls_config_set_cert_file(cfg, c->cert_file) == -1)
+        goto err_config;
+    if (c->key_file && tls_config_set_key_file(cfg, c->key_file) == -1)
+        goto err_config;
+    if (!c->verify) {
+        tls_config_insecure_noverifycert(cfg);
+        tls_config_insecure_noverifyname(cfg);
+        tls_config_insecure_noverifytime(cfg);
+    }
+    if (tls_configure(p->ctx, cfg) == -1)
+        goto err_ctx;
+
+    if (!c->listen) {
+        ret = tls_connect_cbs(p->ctx, tls_read_callback, tls_write_callback,
+            c->tcp, !c->numerichost ? c->host : NULL);
+    } else {
+        struct tls *ctx_new;
+        ret = tls_accept_cbs(p->ctx, &ctx_new, tls_read_callback,
+            tls_write_callback, c->tcp);
+        if (ret == 0) {
+            // free "server" context and replace by "connection" context
+            tls_free(p->ctx);
+            p->ctx = ctx_new;
+        }
+    }
+    if (ret == -1)
+        goto err_ctx;
+
+    tls_config_free(cfg);
+    return 0;
+err_config:
+    av_log(h, AV_LOG_ERROR, "%s\n", tls_config_error(cfg));
+    ret = AVERROR(EIO);
+    goto fail;
+err_ctx:
+    av_log(h, AV_LOG_ERROR, "%s\n", tls_error(p->ctx));
+    ret = AVERROR(EIO);
+    /* fallthrough */
+fail:
+    if (cfg)
+        tls_config_free(cfg);
+    ff_tls_close(h);
+    return ret;
+}
+
+static int ff_tls_read(URLContext *h, uint8_t *buf, int size)
+{
+    TLSContext *p = h->priv_data;
+    ssize_t ret;
+    ret = tls_read(p->ctx, buf, size);
+    if (ret > 0)
+        return ret;
+    else if (ret == 0)
+        return AVERROR_EOF;
+    av_log(h, AV_LOG_ERROR, "%s\n", tls_error(p->ctx));
+    return AVERROR(EIO);
+}
+
+static int ff_tls_write(URLContext *h, const uint8_t *buf, int size)
+{
+    TLSContext *p = h->priv_data;
+    ssize_t ret;
+    ret = tls_write(p->ctx, buf, size);
+    if (ret > 0)
+        return ret;
+    else if (ret == 0)
+        return AVERROR_EOF;
+    av_log(h, AV_LOG_ERROR, "%s\n", tls_error(p->ctx));
+    return AVERROR(EIO);
+}
+
+static int tls_get_file_handle(URLContext *h)
+{
+    TLSContext *c = h->priv_data;
+    return ffurl_get_file_handle(c->tls_shared.tcp);
+}
+
+static const AVOption options[] = {
+    TLS_COMMON_OPTIONS(TLSContext, tls_shared),
+    { NULL }
+};
+
+static const AVClass tls_class = {
+    .class_name = "tls",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+const URLProtocol ff_tls_protocol = {
+    .name           = "tls",
+    .url_open2      = ff_tls_open,
+    .url_read       = ff_tls_read,
+    .url_write      = ff_tls_write,
+    .url_close      = ff_tls_close,
+    .url_get_file_handle = tls_get_file_handle,
+    .priv_data_size = sizeof(TLSContext),
+    .flags          = URL_PROTOCOL_FLAG_NETWORK,
+    .priv_data_class = &tls_class,
+};