From cd2f8a22e94700c68b1de7968df11e8bebfd315b Mon Sep 17 00:00:00 2001 From: Hao Guan Date: Fri, 21 Jun 2024 14:08:20 +0800 Subject: [PATCH] avcodec/videotoolboxenc: fix vtctx reset condition In vtenc_populate_extradata, the cleanup function vtenc_reset should not be used when no error occurs, otherwise some color information is lost (#11036). This patch checks the status code and conducts the correct cleanup. Signed-off-by: Hao Guan Signed-off-by: Zhao Zhili --- libavcodec/videotoolboxenc.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c index 42065cbcb3..bc505abee4 100644 --- a/libavcodec/videotoolboxenc.c +++ b/libavcodec/videotoolboxenc.c @@ -2767,7 +2767,14 @@ static int vtenc_populate_extradata(AVCodecContext *avctx, pe_cleanup: CVPixelBufferRelease(pix_buf); - vtenc_reset(vtctx); + + if (status) { + vtenc_reset(vtctx); + } else if (vtctx->session) { + CFRelease(vtctx->session); + vtctx->session = NULL; + } + vtctx->frame_ct_out = 0; av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));