diff --git a/doc/filters.texi b/doc/filters.texi
index b957b9302e..70fd7a4cc7 100644
--- a/doc/filters.texi
+++ b/doc/filters.texi
@@ -15485,6 +15485,11 @@ Xmap and Ymap input video streams are 16bit depth, single channel.
 @item format
 Specify pixel format of output from this filter. Can be @code{color} or @code{gray}.
 Default is @code{color}.
+
+@item fill
+Specify the color of the unmapped pixels. For the syntax of this option,
+check the @ref{color syntax,,"Color" section in the ffmpeg-utils
+manual,ffmpeg-utils}. Default color is @code{black}.
 @end table
 
 @section removegrain
diff --git a/libavfilter/vf_remap.c b/libavfilter/vf_remap.c
index 7576b9991f..6d5d75225b 100644
--- a/libavfilter/vf_remap.c
+++ b/libavfilter/vf_remap.c
@@ -36,10 +36,12 @@
  * Target_frame[y][x] = Source_frame[ ymap[y][x] ][ [xmap[y][x] ];
  */
 
+#include "libavutil/colorspace.h"
 #include "libavutil/imgutils.h"
 #include "libavutil/pixdesc.h"
 #include "libavutil/opt.h"
 #include "avfilter.h"
+#include "drawutils.h"
 #include "formats.h"
 #include "framesync.h"
 #include "internal.h"
@@ -52,6 +54,8 @@ typedef struct RemapContext {
     int nb_planes;
     int nb_components;
     int step;
+    uint8_t fill_rgba[4];
+    int fill_color[4];
 
     FFFrameSync fs;
 
@@ -65,6 +69,7 @@ static const AVOption remap_options[] = {
     { "format", "set output format", OFFSET(format), AV_OPT_TYPE_INT, {.i64=0}, 0, 1, FLAGS, "format" },
         { "color",  "", 0, AV_OPT_TYPE_CONST, {.i64=0},   .flags = FLAGS, .unit = "format" },
         { "gray",   "", 0, AV_OPT_TYPE_CONST, {.i64=1},   .flags = FLAGS, .unit = "format" },
+    { "fill", "set the color of the unmapped pixels", OFFSET(fill_rgba), AV_OPT_TYPE_COLOR, {.str="black"}, .flags = FLAGS },
     { NULL }
 };
 
@@ -140,6 +145,7 @@ fail:
 static int remap_planar##bits##_##name##_slice(AVFilterContext *ctx, void *arg,             \
                                                int jobnr, int nb_jobs)                      \
 {                                                                                           \
+    RemapContext *s = ctx->priv;                                                            \
     const ThreadData *td = arg;                                                             \
     const AVFrame *in  = td->in;                                                            \
     const AVFrame *xin = td->xin;                                                           \
@@ -158,13 +164,14 @@ static int remap_planar##bits##_##name##_slice(AVFilterContext *ctx, void *arg,
         const int slinesize  = in->linesize[plane] / div;                                   \
         const uint16_t *xmap = (const uint16_t *)xin->data[0] + slice_start * xlinesize;    \
         const uint16_t *ymap = (const uint16_t *)yin->data[0] + slice_start * ylinesize;    \
+        const int color = s->fill_color[plane];                                             \
                                                                                             \
         for (y = slice_start; y < slice_end; y++) {                                         \
             for (x = 0; x < out->width; x++) {                                              \
                 if (ymap[x] < in->height && xmap[x] < in->width) {                          \
                     dst[x] = src[ymap[x] * slinesize + xmap[x]];                            \
                 } else {                                                                    \
-                    dst[x] = 0;                                                             \
+                    dst[x] = color;                                                         \
                 }                                                                           \
             }                                                                               \
             dst  += dlinesize;                                                              \
@@ -189,6 +196,7 @@ DEFINE_REMAP_PLANAR_FUNC(nearest, 16, 2)
 static int remap_packed##bits##_##name##_slice(AVFilterContext *ctx, void *arg,             \
                                                int jobnr, int nb_jobs)                      \
 {                                                                                           \
+    RemapContext *s = ctx->priv;                                                            \
     const ThreadData *td = arg;                                                             \
     const AVFrame *in  = td->in;                                                            \
     const AVFrame *xin = td->xin;                                                           \
@@ -213,7 +221,7 @@ static int remap_packed##bits##_##name##_slice(AVFilterContext *ctx, void *arg,
                 if (ymap[x] < in->height && xmap[x] < in->width) {                          \
                     dst[x * step + c] = src[ymap[x] * slinesize + xmap[x] * step + c];      \
                 } else {                                                                    \
-                    dst[x * step + c] = 0;                                                  \
+                    dst[x * step + c] = s->fill_color[c];                                   \
                 }                                                                           \
             }                                                                               \
         }                                                                                   \
@@ -233,11 +241,28 @@ static int config_input(AVFilterLink *inlink)
     AVFilterContext *ctx = inlink->dst;
     RemapContext *s = ctx->priv;
     const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format);
+    int depth = desc->comp[0].depth;
+    int is_rgb = !!(desc->flags & AV_PIX_FMT_FLAG_RGB);
+    int factor = 1 << (depth - 8);
+    uint8_t rgba_map[4];
 
+    ff_fill_rgba_map(rgba_map, inlink->format);
     s->nb_planes = av_pix_fmt_count_planes(inlink->format);
     s->nb_components = desc->nb_components;
 
-    if (desc->comp[0].depth == 8) {
+    if (is_rgb) {
+        s->fill_color[rgba_map[0]] = s->fill_rgba[0] * factor;
+        s->fill_color[rgba_map[1]] = s->fill_rgba[1] * factor;
+        s->fill_color[rgba_map[2]] = s->fill_rgba[2] * factor;
+        s->fill_color[rgba_map[3]] = s->fill_rgba[3] * factor;
+    } else {
+        s->fill_color[0] = RGB_TO_Y_BT709(s->fill_rgba[0], s->fill_rgba[1], s->fill_rgba[2]) * factor;
+        s->fill_color[1] = RGB_TO_U_BT709(s->fill_rgba[0], s->fill_rgba[1], s->fill_rgba[2], 0) * factor;
+        s->fill_color[2] = RGB_TO_V_BT709(s->fill_rgba[0], s->fill_rgba[1], s->fill_rgba[2], 0) * factor;
+        s->fill_color[3] = s->fill_rgba[3] * factor;
+    }
+
+    if (depth == 8) {
         if (s->nb_planes > 1 || s->nb_components == 1) {
             s->remap_slice = remap_planar8_nearest_slice;
         } else {