diff --git a/libavutil/hwcontext_dxva2.c b/libavutil/hwcontext_dxva2.c
index 89fedd2cd5523c842b3656728f2a5a51eb819ee4..4ed0d56aea7282e50e3a0edaa2a142eba1094e70 100644
--- a/libavutil/hwcontext_dxva2.c
+++ b/libavutil/hwcontext_dxva2.c
@@ -331,6 +331,9 @@ static int dxva2_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
     AVFrame *map;
     int ret;
 
+    if (src->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);
@@ -355,6 +358,9 @@ static int dxva2_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
     ptrdiff_t src_linesize[4], dst_linesize[4];
     int ret, i;
 
+    if (dst->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);