diff --git a/libavutil/hwcontext_dxva2.c b/libavutil/hwcontext_dxva2.c
index 3fe19a6b650466787a0ccbf54b8c451e4e58ed7c..ccf03c8e9ff9b6e2d1405cfdf10b03c273a25b2d 100644
--- a/libavutil/hwcontext_dxva2.c
+++ b/libavutil/hwcontext_dxva2.c
@@ -315,6 +315,9 @@ static int dxva2_transfer_data_to(AVHWFramesContext *ctx, AVFrame *dst,
     AVFrame *map;
     int ret;
 
+    if (src->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);
@@ -339,6 +342,9 @@ static int dxva2_transfer_data_from(AVHWFramesContext *ctx, AVFrame *dst,
     ptrdiff_t src_linesize[4], dst_linesize[4];
     int ret, i;
 
+    if (dst->format != ctx->sw_format)
+        return AVERROR(ENOSYS);
+
     map = av_frame_alloc();
     if (!map)
         return AVERROR(ENOMEM);