From d4202809dac0a13a1eb37165e70405c3c19e03f1 Mon Sep 17 00:00:00 2001 From: Niklas Haas Date: Sat, 21 Feb 2026 15:44:16 +0100 Subject: [PATCH] swscale/format: check pixel type in ff_sws_encode/decode_colors() This would otherwise generate illegal ops and undefined behavior, for pixel formats without any supported corresponding pixel type. This didn't cause any issues previously because the following `ff_sws_encode_pixfmt` would normally fail for such formats, and the UB was ignored in practice. Signed-off-by: Niklas Haas --- libswscale/format.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libswscale/format.c b/libswscale/format.c index db3bf78a72..10cecc8ae4 100644 --- a/libswscale/format.c +++ b/libswscale/format.c @@ -1319,10 +1319,13 @@ int ff_sws_decode_colors(SwsContext *ctx, SwsPixelType type, SwsOpList *ops, const SwsFormat *fmt, bool *incomplete) { const AVLumaCoefficients *c = av_csp_luma_coeffs_from_avcsp(fmt->csp); + const SwsPixelType pixel_type = fmt_pixel_type(fmt->format); + if (!pixel_type) + return AVERROR(ENOTSUP); RET(ff_sws_op_list_append(ops, &(SwsOp) { .op = SWS_OP_CONVERT, - .type = fmt_pixel_type(fmt->format), + .type = pixel_type, .convert.to = type, })); @@ -1402,6 +1405,9 @@ int ff_sws_encode_colors(SwsContext *ctx, SwsPixelType type, const SwsFormat *dst, bool *incomplete) { const AVLumaCoefficients *c = av_csp_luma_coeffs_from_avcsp(dst->csp); + const SwsPixelType pixel_type = fmt_pixel_type(dst->format); + if (!pixel_type) + return AVERROR(ENOTSUP); switch (dst->csp) { case AVCOL_SPC_RGB: @@ -1496,7 +1502,7 @@ int ff_sws_encode_colors(SwsContext *ctx, SwsPixelType type, return ff_sws_op_list_append(ops, &(SwsOp) { .type = type, .op = SWS_OP_CONVERT, - .convert.to = fmt_pixel_type(dst->format), + .convert.to = pixel_type, }); }