From: Daniel Golle <daniel@makrotopia.org>
To: Detlev Casanova <detlev.casanova@collabora.com>,
Ezequiel Garcia <ezequiel@vanguardiasur.com.ar>,
Mauro Carvalho Chehab <mchehab@kernel.org>,
Heiko Stuebner <heiko@sntech.de>,
Nicolas Dufresne <nicolas.dufresne@collabora.com>,
Alex Bee <knaerzche@gmail.com>,
Hans Verkuil <hverkuil+cisco@kernel.org>,
Jonas Karlman <jonas@kwiboo.se>,
Sebastian Fricke <sebastian.fricke@collabora.com>,
linux-media@vger.kernel.org, linux-rockchip@lists.infradead.org,
linux-arm-kernel@lists.infradead.org,
linux-kernel@vger.kernel.org
Subject: [PATCH 7/7] media: rkvdec: hevc: check the RPS scratch allocation
Date: Sat, 14 Nov 2026 10:23:22 +0000 [thread overview]
Message-ID: <e09a3368475ce68d7498a7ff3521e2da6c1f74ea.1794645268.git.daniel@makrotopia.org> (raw)
In-Reply-To: <cover.1794645268.git.daniel@makrotopia.org>
rkvdec_hevc_prepare_hw_st_rps() uses the array returned by kzalloc()
without checking it, so a failed allocation while a decode job is
prepared dereferences a NULL pointer. Return -ENOMEM through
rkvdec_hevc_assemble_hw_rps() and let both back ends refuse the job
after completing the request's controls.
Fixes: c9a59dc2acc7 ("media: rkvdec: Add HEVC support for the VDPU381 variant")
Signed-off-by: Daniel Golle <daniel@makrotopia.org>
---
.../rockchip/rkvdec/rkvdec-hevc-common.c | 26 ++++++++++++++-----
.../rockchip/rkvdec/rkvdec-hevc-common.h | 4 +--
.../rockchip/rkvdec/rkvdec-vdpu381-hevc.c | 7 ++++-
.../rockchip/rkvdec/rkvdec-vdpu383-hevc.c | 7 ++++-
4 files changed, 33 insertions(+), 11 deletions(-)
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
index e3449ab0d415b..80df05bacaeba 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.c
@@ -329,24 +329,27 @@ static void st_ref_pic_set_calculate(struct rkvdec_hevc_run *run, int idx,
st_rps->num_delta_pocs = st_rps->num_positive_pics + st_rps->num_negative_pics;
}
-static void rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
- struct rkvdec_hevc_st_rps_cache *cache)
+static int rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+ struct rkvdec_hevc_st_rps_cache *cache)
{
u8 num_sets = run->sps->num_short_term_ref_pic_sets;
size_t size;
int idx;
if (!run->ext_sps_st_rps)
- return;
+ return 0;
size = num_sets * sizeof(*cache->sets);
if (cache->num_sets == num_sets && !memcmp(cache->sets, run->ext_sps_st_rps, size))
- return;
+ return 0;
struct calculated_rps_st_set *calculated_rps_st_sets =
kzalloc(sizeof(struct calculated_rps_st_set) *
run->sps->num_short_term_ref_pic_sets, GFP_KERNEL);
+ if (!calculated_rps_st_sets)
+ return -ENOMEM;
+
for (idx = 0; idx < run->sps->num_short_term_ref_pic_sets; idx++) {
const struct v4l2_ctrl_hevc_ext_sps_st_rps *rps_data = &run->ext_sps_st_rps[idx];
@@ -362,13 +365,22 @@ static void rkvdec_hevc_prepare_hw_st_rps(struct rkvdec_hevc_run *run, struct rk
memcpy(cache->sets, run->ext_sps_st_rps, size);
cache->num_sets = num_sets;
+
+ return 0;
}
-void rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
- struct rkvdec_hevc_st_rps_cache *st_cache)
+int rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+ struct rkvdec_hevc_st_rps_cache *st_cache)
{
- rkvdec_hevc_prepare_hw_st_rps(run, rps, st_cache);
+ int ret;
+
+ ret = rkvdec_hevc_prepare_hw_st_rps(run, rps, st_cache);
+ if (ret)
+ return ret;
+
rkvdec_hevc_assemble_hw_lt_rps(run, rps);
+
+ return 0;
}
struct vb2_buffer *
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
index f6a3dec6ff4ad..32f192743c8c8 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-hevc-common.h
@@ -71,8 +71,8 @@ void compute_tiles_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size,
void compute_tiles_non_uniform(struct rkvdec_hevc_run *run, u16 log2_min_cb_size,
u16 width, u16 height, s32 pic_in_cts_width,
s32 pic_in_cts_height, u16 *column_width, u16 *row_height);
-void rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
- struct rkvdec_hevc_st_rps_cache *st_cache);
+int rkvdec_hevc_assemble_hw_rps(struct rkvdec_hevc_run *run, struct rkvdec_rps *rps,
+ struct rkvdec_hevc_st_rps_cache *st_cache);
void rkvdec_hevc_assemble_hw_scaling_list(struct rkvdec_ctx *ctx,
struct rkvdec_hevc_run *run,
struct scaling_factor *scaling_factor,
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
index 8388421a68a48..2a239114759a2 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu381-hevc.c
@@ -593,6 +593,7 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
struct rkvdec_hevc_run run;
struct rkvdec_hevc_ctx *hevc_ctx = ctx->priv;
struct rkvdec_hevc_priv_tbl *tbl = hevc_ctx->priv_tbl.cpu;
+ int ret;
rkvdec_hevc_run_preamble(ctx, &run);
@@ -606,7 +607,11 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
rkvdec_hevc_assemble_hw_scaling_list(ctx, &run, &tbl->scaling_list,
&hevc_ctx->scaling_matrix_cache);
assemble_hw_pps(ctx, &run);
- rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+ ret = rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+ if (ret) {
+ rkvdec_run_postamble(ctx, &run.base);
+ return ret;
+ }
config_registers(ctx, &run);
diff --git a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
index 47f236295f1e5..52554c75f47f4 100644
--- a/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
+++ b/drivers/media/platform/rockchip/rkvdec/rkvdec-vdpu383-hevc.c
@@ -542,6 +542,7 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
struct rkvdec_hevc_ctx *hevc_ctx = ctx->priv;
struct rkvdec_hevc_priv_tbl *tbl = hevc_ctx->priv_tbl.cpu;
u32 timeout_threshold;
+ int ret;
rkvdec_hevc_run_preamble(ctx, &run);
@@ -560,7 +561,11 @@ static int rkvdec_hevc_run(struct rkvdec_ctx *ctx)
rkvdec_hevc_assemble_hw_scaling_list(ctx, &run, &tbl->scaling_list,
&hevc_ctx->scaling_matrix_cache);
assemble_hw_pps(ctx, &run);
- rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+ ret = rkvdec_hevc_assemble_hw_rps(&run, &tbl->rps, &hevc_ctx->st_cache);
+ if (ret) {
+ rkvdec_run_postamble(ctx, &run.base);
+ return ret;
+ }
config_registers(ctx, &run);
--
2.56.0
prev parent reply other threads:[~2026-11-14 10:23 UTC|newest]
Thread overview: 7+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-14 10:22 [PATCH 0/7] media: rkvdec: Fix the HEVC reference picture set handling Daniel Golle
2026-11-14 10:22 ` [PATCH 1/7] media: rkvdec: hevc: apply request controls first Daniel Golle
2026-11-14 10:22 ` [PATCH 2/7] media: rkvdec: vdpu383: complete refused HEVC requests Daniel Golle
2026-11-14 10:22 ` [PATCH 3/7] media: rkvdec: hevc: bound the RPS reads by the control size Daniel Golle
2026-11-14 10:23 ` [PATCH 5/7] media: rkvdec: vdpu381: refuse HEVC jobs without RPS sets Daniel Golle
2026-11-14 10:23 ` [PATCH 6/7] media: rkvdec: hevc: cache every short-term RPS set Daniel Golle
2026-11-14 10:23 ` Daniel Golle [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=e09a3368475ce68d7498a7ff3521e2da6c1f74ea.1794645268.git.daniel@makrotopia.org \
--to=daniel@makrotopia.org \
--cc=detlev.casanova@collabora.com \
--cc=ezequiel@vanguardiasur.com.ar \
--cc=heiko@sntech.de \
--cc=hverkuil+cisco@kernel.org \
--cc=jonas@kwiboo.se \
--cc=knaerzche@gmail.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-media@vger.kernel.org \
--cc=linux-rockchip@lists.infradead.org \
--cc=mchehab@kernel.org \
--cc=nicolas.dufresne@collabora.com \
--cc=sebastian.fricke@collabora.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®