From: Charvi Mehta <charvi.mehta@arm.com>
To: linux-media@vger.kernel.org, devicetree@vger.kernel.org,
linux-kernel@vger.kernel.org
Cc: Charvi Mehta <charvi.mehta@arm.com>,
Mauro Carvalho Chehab <mchehab@kernel.org>,
Rob Herring <robh@kernel.org>,
Krzysztof Kozlowski <krzk+dt@kernel.org>,
Conor Dooley <conor+dt@kernel.org>,
Philipp Zabel <p.zabel@pengutronix.de>,
Vincenzo Frascino <vincenzo.frascino@arm.com>,
Rathinavel Sankaralingam <rath.sankaralingam@arm.com>,
Shobhit Pandey <shobhit.pandey@arm.com>
Subject: [RFC PATCH 23/28] media: mali-vpu: propagate buffer metadata
Date: Thu, 19 Nov 2026 15:37:53 +0000 [thread overview]
Message-ID: <20261119153758.1426673-24-charvi.mehta@arm.com> (raw)
In-Reply-To: <20261119153758.1426673-1-charvi.mehta@arm.com>
Firmware may return compressed buffers before producing their
corresponding decoded frames and may reorder decoded output. Preserve
timestamp-associated flags and timecodes across firmware processing and
restore them on the matching CAPTURE buffers.
Report the negotiated field on CAPTURE buffers and maintain independent
sequence counters for the OUTPUT and CAPTURE queues.
Signed-off-by: Charvi Mehta <charvi.mehta@arm.com>
---
.../platform/arm/mali-vpu/mali-vpu-session.c | 74 +++++++++++++++++++
.../platform/arm/mali-vpu/mali-vpu-session.h | 21 ++++++
.../platform/arm/mali-vpu/mali-vpu-vb2.c | 3 +
3 files changed, 98 insertions(+)
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
index 9cbd116124109..ea80aacd930d8 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.c
@@ -3,6 +3,7 @@
* Copyright (C) 2026 ARM Limited.
*/
+#include <linux/array_size.h>
#include <linux/device.h>
#include <linux/err.h>
#include <linux/errno.h>
@@ -19,6 +20,9 @@
#include "mali-vpu-variant.h"
#include "mali-vpu-vb2.h"
+#define MALI_VPU_TS_METADATA_FLAGS \
+ (V4L2_BUF_FLAG_TIMECODE | V4L2_BUF_FLAG_TSTAMP_SRC_MASK)
+
static const unsigned int mali_vpu_fw_response_timeout_ms = 10000;
static const unsigned int mali_vpu_fw_load_timeout_ms = 10000;
@@ -288,12 +292,65 @@ static int mali_vpu_session_sched_activate(struct mali_vpu_session *sess)
return mali_vpu_sched_switch_in(&sess->sched_sess, &sess->vpu_dev->sched);
}
+static void mali_vpu_session_store_ts_metadata(struct mali_vpu_session *sess,
+ const struct vb2_v4l2_buffer *vbuf)
+{
+ struct mali_vpu_ts_metadata *metadata;
+
+ if (sess->ts_metadata_idx >= ARRAY_SIZE(sess->ts_metadata))
+ sess->ts_metadata_idx = 0;
+
+ metadata = &sess->ts_metadata[sess->ts_metadata_idx++];
+ metadata->timestamp = vbuf->vb2_buf.timestamp;
+ metadata->flags = vbuf->flags & MALI_VPU_TS_METADATA_FLAGS;
+ metadata->valid = true;
+ metadata->timecode = vbuf->timecode;
+}
+
+static void mali_vpu_session_restore_ts_metadata(struct mali_vpu_session *sess,
+ struct vb2_v4l2_buffer *vbuf)
+{
+ u64 timestamp = vbuf->vb2_buf.timestamp;
+ u32 i;
+
+ /*
+ * Matching the auxiliary metadata by timestamp is ambiguous when
+ * OUTPUT buffers have duplicate timestamps: timecode and timestamp-
+ * source flags may be restored from a different OUTPUT buffer.
+ *
+ * The timestamp itself is returned independently through firmware.
+ * Unique association tags require reliable metadata lifetime tracking.
+ */
+
+ vbuf->flags &= ~MALI_VPU_TS_METADATA_FLAGS;
+
+ for (i = 0; i < ARRAY_SIZE(sess->ts_metadata); i++) {
+ struct mali_vpu_ts_metadata *metadata = &sess->ts_metadata[i];
+
+ if (!metadata->valid || metadata->timestamp != timestamp)
+ continue;
+
+ vbuf->flags |= metadata->flags;
+ if (metadata->flags & V4L2_BUF_FLAG_TIMECODE)
+ vbuf->timecode = metadata->timecode;
+
+ return;
+ }
+}
+
+void mali_vpu_session_reset_ts_metadata(struct mali_vpu_session *sess)
+{
+ memset(sess->ts_metadata, 0, sizeof(sess->ts_metadata));
+ sess->ts_metadata_idx = 0;
+}
+
static void mali_vpu_session_buffer_done(struct mali_vpu_session *sess,
struct mali_vpu_buffer *vpu_buf,
enum vb2_buffer_state state)
{
struct vb2_v4l2_buffer *vbuf = &vpu_buf->m2m_buf.vb;
struct vb2_buffer *vb2_buf = &vbuf->vb2_buf;
+ struct mali_vpu_port *port;
if (vb2_buf->state != VB2_BUF_STATE_ACTIVE) {
dev_dbg(sess->vpu_dev->dev,
@@ -302,6 +359,14 @@ static void mali_vpu_session_buffer_done(struct mali_vpu_session *sess,
return;
}
+ port = V4L2_TYPE_IS_OUTPUT(vpu_buf->type) ? &sess->port_src : &sess->port_dst;
+
+ vbuf->sequence = port->sequence++;
+ if (V4L2_TYPE_IS_CAPTURE(vpu_buf->type)) {
+ vbuf->field = port->curr_fmt.fmt.pix_mp.field;
+ mali_vpu_session_restore_ts_metadata(sess, vbuf);
+ }
+
if (V4L2_TYPE_IS_OUTPUT(vpu_buf->type))
v4l2_m2m_src_buf_remove_by_buf(sess->m2m_ctx, vbuf);
else
@@ -607,6 +672,10 @@ int mali_vpu_session_qbuf(struct mali_vpu_session *sess, struct vb2_buffer *vb2_
!mali_vpu_session_can_stream(sess) ||
sess->fw_state != MALI_VPU_FW_STATE_RUNNING) {
vpu_buf->attr = MALI_VPU_BUF_ATTR_DEFERRED;
+
+ if (V4L2_TYPE_IS_OUTPUT(vb2_buf->type))
+ mali_vpu_session_store_ts_metadata(sess, v4l2_buf);
+
return 0;
}
@@ -616,6 +685,9 @@ int mali_vpu_session_qbuf(struct mali_vpu_session *sess, struct vb2_buffer *vb2_
vpu_buf->attr = MALI_VPU_BUF_ATTR_QUEUED;
+ if (V4L2_TYPE_IS_OUTPUT(vb2_buf->type))
+ mali_vpu_session_store_ts_metadata(sess, v4l2_buf);
+
mali_vpu_sched_send_irq(&sess->sched_sess, &sess->vpu_dev->sched);
return 0;
@@ -733,6 +805,8 @@ int mali_vpu_session_prepare_streaming(struct mali_vpu_session *sess,
}
}
+ port->sequence = 0;
+
return 0;
err_rollback:
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.h b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.h
index c6e1add34a8f1..7565a680397a8 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-session.h
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-session.h
@@ -29,11 +29,27 @@ struct mali_vpu_sched_session;
* @supported_fmts: List of format descriptors supported by the queue.
* @stream_requested: Port/Queue is being prepared for streaming or has
* successfully completed preparation. Protected by vq_lock.
+ * @sequence: Sequence number assigned to the next completed buffer.
*/
struct mali_vpu_port {
struct v4l2_format curr_fmt;
struct mali_vpu_format_list supported_fmts;
bool stream_requested;
+ u32 sequence;
+};
+
+/**
+ * struct mali_vpu_ts_metadata - OUTPUT metadata propagated to CAPTURE.
+ * @timestamp: Timestamp used as the firmware association tag.
+ * @flags: Timestamp-source and timecode-presence flags.
+ * @valid: Whether this cache entry has been populated.
+ * @timecode: Timecode associated with @timestamp.
+ */
+struct mali_vpu_ts_metadata {
+ u64 timestamp;
+ u32 flags;
+ bool valid;
+ struct v4l2_timecode timecode;
};
/*
@@ -80,6 +96,8 @@ enum mali_vpu_session_fw_bin_state {
* @mmu: MMU handle for this session
* @vq_lock: Mutex lock to serialise per-session queue related ioctls.
* @min_buffers_required: Driver-reported minimum buffers for role-specific queue.
+ * @ts_metadata: OUTPUT metadata indexed by timestamp for CAPTURE propagation.
+ * @ts_metadata_idx: Next timestamp metadata slot to use.
* @ctrls_caps: Variant-provided immutable control capability table.
* @num_ctrls_caps: Number of entries in @ctrls_caps.
* @ctrls_state: Per-session control cache and staging state.
@@ -106,6 +124,8 @@ struct mali_vpu_session {
struct mali_vpu_mmu mmu;
struct mutex vq_lock; /* lock to serialise per-session ioctls */
u32 min_buffers_required;
+ struct mali_vpu_ts_metadata ts_metadata[VIDEO_MAX_FRAME];
+ u32 ts_metadata_idx;
const struct mali_vpu_ctrl_desc *ctrls_caps;
u32 num_ctrls_caps;
@@ -139,6 +159,7 @@ int mali_vpu_session_prepare_streaming(struct mali_vpu_session *sess,
int mali_vpu_session_streamoff(struct mali_vpu_session *sess, enum v4l2_buf_type type);
void mali_vpu_session_irq(struct mali_vpu_session *sess);
int mali_vpu_session_qbuf(struct mali_vpu_session *sess, struct vb2_buffer *vb2_buf);
+void mali_vpu_session_reset_ts_metadata(struct mali_vpu_session *sess);
static inline struct mali_vpu_session *mali_vpu_get_session(struct file *filp)
{
diff --git a/drivers/media/platform/arm/mali-vpu/mali-vpu-vb2.c b/drivers/media/platform/arm/mali-vpu/mali-vpu-vb2.c
index 41158bf4e0ecb..81a447ceeb755 100644
--- a/drivers/media/platform/arm/mali-vpu/mali-vpu-vb2.c
+++ b/drivers/media/platform/arm/mali-vpu/mali-vpu-vb2.c
@@ -137,6 +137,9 @@ static void mali_vpu_vb2_unprepare_streaming(struct vb2_queue *q)
if (!vb2_start_streaming_called(q))
v4l2_m2m_update_stop_streaming_state(sess->m2m_ctx, q);
+
+ if (!sess->port_src.stream_requested && !sess->port_dst.stream_requested)
+ mali_vpu_session_reset_ts_metadata(sess);
}
static const struct vb2_ops mali_vpu_vb2_ops = {
--
2.43.0
next prev parent reply other threads:[~2026-11-19 15:39 UTC|newest]
Thread overview: 29+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-19 15:37 [RFC PATCH 00/28] media: add arm Mali-V76 video decoder driver Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 01/28] dt-bindings: media: add Arm Mali-V76 VPU Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 02/28] media: mali-vpu: add Mali VPU platform driver Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 03/28] media: mali-vpu: register V4L2 decoder device and add querycap ioctl Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 04/28] media: mali-vpu: initialize decoder format state Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 05/28] media: mali-vpu: add V4L2 mem2mem and VB2 queue support Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 06/28] media: mali-vpu: implement format query ioctls Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 07/28] media: mali-vpu: implement TRY_FMT and S_FMT ioctls Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 08/28] media: mali-vpu: register V76 decoder controls and introduce UAPI controls Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 09/28] media: mali-vpu: add MMU software page table manager Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 10/28] media: mali-vpu: add firmware cache infrastructure Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 11/28] media: mali-vpu: load firmware for decoder sessions Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 12/28] media: mali-vpu: prepare firmware address space Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 13/28] media: mali-vpu: add firmware message transport Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 14/28] media: mali-vpu: route decoder controls to firmware Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 15/28] media: mali-vpu: handle firmware RPC requests Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 16/28] media: mali-vpu: add LSID hardware support Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 17/28] media: mali-vpu: add single session scheduler Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 18/28] media: mali-vpu: handle firmware interrupts Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 19/28] media: mali-vpu: start decoder firmware through LSID scheduling Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 20/28] media: mali-vpu: implement vb2 buffer queue handling Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 21/28] media: mali-vpu: handle firmware buffer exchange Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 22/28] media: mali-vpu: handle fatal firmware error Charvi Mehta
2026-11-19 15:37 ` Charvi Mehta [this message]
2026-11-19 15:37 ` [RFC PATCH 24/28] media: mali-vpu: add decoder EOS and DRC handling Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 25/28] media: mali-vpu: handle firmware buffer parameters Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 26/28] media: mali-vpu: handle firmware event responses Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 27/28] media: mali-vpu: add stateful decoder state machine Charvi Mehta
2026-11-19 15:37 ` [RFC PATCH 28/28] media: MAINTAINERS: add Arm Mali-V76 VPU driver Charvi Mehta
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119153758.1426673-24-charvi.mehta@arm.com \
--to=charvi.mehta@arm.com \
--cc=conor+dt@kernel.org \
--cc=devicetree@vger.kernel.org \
--cc=krzk+dt@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-media@vger.kernel.org \
--cc=mchehab@kernel.org \
--cc=p.zabel@pengutronix.de \
--cc=rath.sankaralingam@arm.com \
--cc=robh@kernel.org \
--cc=shobhit.pandey@arm.com \
--cc=vincenzo.frascino@arm.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®