vaapi_encode.c

    for (i = 0; i < FF_ARRAY_ELEMS(attr); i++) {
        if (attr[i].value == VA_ATTRIB_NOT_SUPPORTED) {
            // Unfortunately we have to treat this as "don't know" and hope
            // for the best, because the Intel MJPEG encoder returns this
            // for all the interesting attributes.
            continue;
        }
        switch (attr[i].type) {
        case VAConfigAttribRTFormat:
            if (!(ctx->va_rt_format & attr[i].value)) {
                av_log(avctx, AV_LOG_ERROR, "Surface RT format %#x "
                       "is not supported (mask %#x).\n",
                       ctx->va_rt_format, attr[i].value);
                err = AVERROR(EINVAL);
                goto fail;
            }
            ctx->config_attributes[ctx->nb_config_attributes++] =
                (VAConfigAttrib) {
                .type  = VAConfigAttribRTFormat,
                .value = ctx->va_rt_format,
            };
            break;
        case VAConfigAttribRateControl:
            if (!(ctx->va_rc_mode & attr[i].value)) {
                av_log(avctx, AV_LOG_ERROR, "Rate control mode %#x "
                       "is not supported (mask: %#x).\n",
                       ctx->va_rc_mode, attr[i].value);
                err = AVERROR(EINVAL);
                goto fail;
            }
            ctx->config_attributes[ctx->nb_config_attributes++] =
                (VAConfigAttrib) {
                .type  = VAConfigAttribRateControl,
                .value = ctx->va_rc_mode,
            };
            break;
        case VAConfigAttribEncMaxRefFrames:
        {
            unsigned int ref_l0 = attr[i].value & 0xffff;
            unsigned int ref_l1 = (attr[i].value >> 16) & 0xffff;

            if (avctx->gop_size > 1 && ref_l0 < 1) {
                av_log(avctx, AV_LOG_ERROR, "P frames are not "
                       "supported (%#x).\n", attr[i].value);
                err = AVERROR(EINVAL);
                goto fail;
            }
            if (avctx->max_b_frames > 0 && ref_l1 < 1) {
                av_log(avctx, AV_LOG_ERROR, "B frames are not "
                       "supported (%#x).\n", attr[i].value);
                err = AVERROR(EINVAL);
                goto fail;
            }
        }
        break;
        case VAConfigAttribEncPackedHeaders:
            if (ctx->va_packed_headers & ~attr[i].value) {
                // This isn't fatal, but packed headers are always
                // preferable because they are under our control.
                // When absent, the driver is generating them and some
                // features may not work (e.g. VUI or SEI in H.264).
                av_log(avctx, AV_LOG_WARNING, "Warning: some packed "
                       "headers are not supported (want %#x, got %#x).\n",
                       ctx->va_packed_headers, attr[i].value);
                ctx->va_packed_headers &= attr[i].value;
            }
            ctx->config_attributes[ctx->nb_config_attributes++] =
                (VAConfigAttrib) {
                .type  = VAConfigAttribEncPackedHeaders,
                .value = ctx->va_packed_headers,
            };
            break;
        default:
            av_assert0(0 && "Unexpected config attribute.");
        }
    }

    err = 0;
fail:
    av_freep(&profiles);
    av_freep(&entrypoints);
    return err;
}

static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
{
    VAAPIEncodeContext *ctx = avctx->priv_data;
    int hrd_buffer_size;
    int hrd_initial_buffer_fullness;

    if (avctx->bit_rate > INT32_MAX) {
        av_log(avctx, AV_LOG_ERROR, "Target bitrate of 2^31 bps or "
               "higher is not supported.\n");
        return AVERROR(EINVAL);
    }

    if (avctx->rc_buffer_size)
        hrd_buffer_size = avctx->rc_buffer_size;
    else
        hrd_buffer_size = avctx->bit_rate;
    if (avctx->rc_initial_buffer_occupancy)
        hrd_initial_buffer_fullness = avctx->rc_initial_buffer_occupancy;
    else
        hrd_initial_buffer_fullness = hrd_buffer_size * 3 / 4;

    ctx->rc_params.misc.type = VAEncMiscParameterTypeRateControl;
    ctx->rc_params.rc = (VAEncMiscParameterRateControl) {
        .bits_per_second   = avctx->bit_rate,
        .target_percentage = 66,
        .window_size       = 1000,
        .initial_qp        = (avctx->qmax >= 0 ? avctx->qmax : 40),
        .min_qp            = (avctx->qmin >= 0 ? avctx->qmin : 18),
        .basic_unit_size   = 0,
    };
    ctx->global_params[ctx->nb_global_params] =
        &ctx->rc_params.misc;
    ctx->global_params_size[ctx->nb_global_params++] =
        sizeof(ctx->rc_params);

    ctx->hrd_params.misc.type = VAEncMiscParameterTypeHRD;
    ctx->hrd_params.hrd = (VAEncMiscParameterHRD) {
        .initial_buffer_fullness = hrd_initial_buffer_fullness,
        .buffer_size             = hrd_buffer_size,
    };
    ctx->global_params[ctx->nb_global_params] =
        &ctx->hrd_params.misc;
    ctx->global_params_size[ctx->nb_global_params++] =
        sizeof(ctx->hrd_params);

    return 0;
}

static void vaapi_encode_free_output_buffer(void *opaque,
                                            uint8_t *data)
{
    AVCodecContext   *avctx = opaque;
    VAAPIEncodeContext *ctx = avctx->priv_data;
    VABufferID buffer_id;

    buffer_id = (VABufferID)(uintptr_t)data;

    vaDestroyBuffer(ctx->hwctx->display, buffer_id);

    av_log(avctx, AV_LOG_DEBUG, "Freed output buffer %#x\n", buffer_id);
}

static AVBufferRef *vaapi_encode_alloc_output_buffer(void *opaque,
                                                     int size)
{
    AVCodecContext   *avctx = opaque;
    VAAPIEncodeContext *ctx = avctx->priv_data;
    VABufferID buffer_id;
    VAStatus vas;
    AVBufferRef *ref;

    // The output buffer size is fixed, so it needs to be large enough
    // to hold the largest possible compressed frame.  We assume here
    // that the uncompressed frame plus some header data is an upper
    // bound on that.
    vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
                         VAEncCodedBufferType,
                         3 * ctx->surface_width * ctx->surface_height +
                         (1 << 16), 1, 0, &buffer_id);
    if (vas != VA_STATUS_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Failed to create bitstream "
               "output buffer: %d (%s).\n", vas, vaErrorStr(vas));
        return NULL;
    }

    av_log(avctx, AV_LOG_DEBUG, "Allocated output buffer %#x\n", buffer_id);

    ref = av_buffer_create((uint8_t*)(uintptr_t)buffer_id,
                           sizeof(buffer_id),
                           &vaapi_encode_free_output_buffer,
                           avctx, AV_BUFFER_FLAG_READONLY);
    if (!ref) {
        vaDestroyBuffer(ctx->hwctx->display, buffer_id);
        return NULL;
    }

    return ref;
}

static av_cold int vaapi_encode_create_recon_frames(AVCodecContext *avctx)
{
    VAAPIEncodeContext *ctx = avctx->priv_data;
    AVVAAPIHWConfig *hwconfig = NULL;
    AVHWFramesConstraints *constraints = NULL;
    enum AVPixelFormat recon_format;
    int err, i;

    hwconfig = av_hwdevice_hwconfig_alloc(ctx->device_ref);
    if (!hwconfig) {
        err = AVERROR(ENOMEM);
        goto fail;
    }
    hwconfig->config_id = ctx->va_config;

    constraints = av_hwdevice_get_hwframe_constraints(ctx->device_ref,
                                                      hwconfig);
    if (!constraints) {
        err = AVERROR(ENOMEM);
        goto fail;
    }

    // Probably we can use the input surface format as the surface format
    // of the reconstructed frames.  If not, we just pick the first (only?)
    // format in the valid list and hope that it all works.
    recon_format = AV_PIX_FMT_NONE;
    if (constraints->valid_sw_formats) {
        for (i = 0; constraints->valid_sw_formats[i] != AV_PIX_FMT_NONE; i++) {
            if (ctx->input_frames->sw_format ==
                constraints->valid_sw_formats[i]) {
                recon_format = ctx->input_frames->sw_format;
                break;
            }
        }
        if (recon_format == AV_PIX_FMT_NONE) {
            // No match.  Just use the first in the supported list and
            // hope for the best.
            recon_format = constraints->valid_sw_formats[0];
        }
    } else {
        // No idea what to use; copy input format.
        recon_format = ctx->input_frames->sw_format;
    }
    av_log(avctx, AV_LOG_DEBUG, "Using %s as format of "
           "reconstructed frames.\n", av_get_pix_fmt_name(recon_format));

    if (ctx->surface_width  < constraints->min_width  ||
        ctx->surface_height < constraints->min_height ||
        ctx->surface_width  > constraints->max_width ||
        ctx->surface_height > constraints->max_height) {
        av_log(avctx, AV_LOG_ERROR, "Hardware does not support encoding at "
               "size %dx%d (constraints: width %d-%d height %d-%d).\n",
               ctx->surface_width, ctx->surface_height,
               constraints->min_width,  constraints->max_width,
               constraints->min_height, constraints->max_height);
        err = AVERROR(EINVAL);
        goto fail;
    }

    av_freep(&hwconfig);
    av_hwframe_constraints_free(&constraints);

    ctx->recon_frames_ref = av_hwframe_ctx_alloc(ctx->device_ref);
    if (!ctx->recon_frames_ref) {
        err = AVERROR(ENOMEM);
        goto fail;
    }
    ctx->recon_frames = (AVHWFramesContext*)ctx->recon_frames_ref->data;

    ctx->recon_frames->format    = AV_PIX_FMT_VAAPI;
    ctx->recon_frames->sw_format = recon_format;
    ctx->recon_frames->width     = ctx->surface_width;
    ctx->recon_frames->height    = ctx->surface_height;
    ctx->recon_frames->initial_pool_size =
        avctx->max_b_frames + 3;

    err = av_hwframe_ctx_init(ctx->recon_frames_ref);
    if (err < 0) {
        av_log(avctx, AV_LOG_ERROR, "Failed to initialise reconstructed "
               "frame context: %d.\n", err);
        goto fail;
    }

    err = 0;
  fail:
    av_freep(&hwconfig);
    av_hwframe_constraints_free(&constraints);
    return err;
}

av_cold int ff_vaapi_encode_init(AVCodecContext *avctx)
{
    VAAPIEncodeContext *ctx = avctx->priv_data;
    AVVAAPIFramesContext *recon_hwctx = NULL;
    VAStatus vas;
    int err;

    if (!avctx->hw_frames_ctx) {
        av_log(avctx, AV_LOG_ERROR, "A hardware frames reference is "
               "required to associate the encoding device.\n");
        return AVERROR(EINVAL);
    }

    ctx->codec_options = ctx->codec_options_data;

    ctx->va_config  = VA_INVALID_ID;
    ctx->va_context = VA_INVALID_ID;

    ctx->priv_data = av_mallocz(ctx->codec->priv_data_size);
    if (!ctx->priv_data) {
        err = AVERROR(ENOMEM);
        goto fail;
    }

    ctx->input_frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
    if (!ctx->input_frames_ref) {
        err = AVERROR(ENOMEM);
        goto fail;
    }
    ctx->input_frames = (AVHWFramesContext*)ctx->input_frames_ref->data;

    ctx->device_ref = av_buffer_ref(ctx->input_frames->device_ref);
    if (!ctx->device_ref) {
        err = AVERROR(ENOMEM);
        goto fail;
    }
    ctx->device = (AVHWDeviceContext*)ctx->device_ref->data;
    ctx->hwctx = ctx->device->hwctx;

    err = vaapi_encode_config_attributes(avctx);
    if (err < 0)
        goto fail;

    vas = vaCreateConfig(ctx->hwctx->display,
                         ctx->va_profile, ctx->va_entrypoint,
                         ctx->config_attributes, ctx->nb_config_attributes,
                         &ctx->va_config);
    if (vas != VA_STATUS_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Failed to create encode pipeline "
               "configuration: %d (%s).\n", vas, vaErrorStr(vas));
        err = AVERROR(EIO);
        goto fail;
    }

    err = vaapi_encode_create_recon_frames(avctx);
    if (err < 0)
        goto fail;

    recon_hwctx = ctx->recon_frames->hwctx;
    vas = vaCreateContext(ctx->hwctx->display, ctx->va_config,
                          ctx->surface_width, ctx->surface_height,
                          VA_PROGRESSIVE,
                          recon_hwctx->surface_ids,
                          recon_hwctx->nb_surfaces,
                          &ctx->va_context);
    if (vas != VA_STATUS_SUCCESS) {
        av_log(avctx, AV_LOG_ERROR, "Failed to create encode pipeline "
               "context: %d (%s).\n", vas, vaErrorStr(vas));
        err = AVERROR(EIO);
        goto fail;
    }

    ctx->output_buffer_pool =
        av_buffer_pool_init2(sizeof(VABufferID), avctx,
                             &vaapi_encode_alloc_output_buffer, NULL);
    if (!ctx->output_buffer_pool) {
        err = AVERROR(ENOMEM);
        goto fail;
    }

    if (ctx->va_rc_mode & ~VA_RC_CQP) {
        err = vaapi_encode_init_rate_control(avctx);
        if (err < 0)
            goto fail;
    }

    if (ctx->codec->configure) {
        err = ctx->codec->configure(avctx);
        if (err < 0)
            goto fail;
    }

    ctx->input_order  = 0;
    ctx->output_delay = avctx->max_b_frames;
    ctx->decode_delay = 1;
    ctx->output_order = - ctx->output_delay - 1;

    if (ctx->codec->sequence_params_size > 0) {
        ctx->codec_sequence_params =
            av_mallocz(ctx->codec->sequence_params_size);
        if (!ctx->codec_sequence_params) {
            err = AVERROR(ENOMEM);
            goto fail;
        }
    }
    if (ctx->codec->picture_params_size > 0) {
        ctx->codec_picture_params =
            av_mallocz(ctx->codec->picture_params_size);
        if (!ctx->codec_picture_params) {
            err = AVERROR(ENOMEM);
            goto fail;
        }
    }

    if (ctx->codec->init_sequence_params) {
        err = ctx->codec->init_sequence_params(avctx);
        if (err < 0) {
            av_log(avctx, AV_LOG_ERROR, "Codec sequence initialisation "
                   "failed: %d.\n", err);
            goto fail;
        }
    }

    // All I are IDR for now.
    ctx->i_per_idr = 0;
    ctx->p_per_i = ((avctx->gop_size + avctx->max_b_frames) /
                    (avctx->max_b_frames + 1));
    ctx->b_per_p = avctx->max_b_frames;

    // This should be configurable somehow.  (Needs testing on a machine
    // where it actually overlaps properly, though.)
    ctx->issue_mode = ISSUE_MODE_MAXIMISE_THROUGHPUT;

    return 0;

fail:
    ff_vaapi_encode_close(avctx);
    return err;
}

av_cold int ff_vaapi_encode_close(AVCodecContext *avctx)
{
    VAAPIEncodeContext *ctx = avctx->priv_data;
    VAAPIEncodePicture *pic, *next;

    for (pic = ctx->pic_start; pic; pic = next) {
        next = pic->next;
        vaapi_encode_free(avctx, pic);
    }

    if (ctx->va_context != VA_INVALID_ID) {
        vaDestroyContext(ctx->hwctx->display, ctx->va_context);
        ctx->va_context = VA_INVALID_ID;
    }

    if (ctx->va_config != VA_INVALID_ID) {
        vaDestroyConfig(ctx->hwctx->display, ctx->va_config);
        ctx->va_config = VA_INVALID_ID;
    }

    av_buffer_pool_uninit(&ctx->output_buffer_pool);

    av_freep(&ctx->codec_sequence_params);
    av_freep(&ctx->codec_picture_params);

    av_buffer_unref(&ctx->recon_frames_ref);
    av_buffer_unref(&ctx->input_frames_ref);
    av_buffer_unref(&ctx->device_ref);

    av_freep(&ctx->priv_data);

    return 0;
}