[PATCH] accel/rocket: Report per-core engine usage through fdinfo

[PATCH] accel/rocket: Report per-core engine usage through fdinfo

[Sidong Yang]

There is no way to tell how much of the NPU a client uses, or whether its
jobs reach more than one core. Implement the DRM usage stats: account the
time each job occupies a core and report it per client as
drm-engine-npu<n>, along with drm_show_memory_stats() for the BOs.

Jobs can outlive the file that submitted them, so the counters are
refcounted and each job holds a reference. The time covers the whole
period a job owns a core, so it also includes handling the completion
interrupt and the gaps between the tasks of a multi-task job.

For example, teflon running a model two operations deep 3001 times gives:

  drm-engine-npu0:	22719385 ns
  drm-engine-npu1:	23345296 ns
  drm-engine-npu2:	0 ns

The NPU is busy 15.6% of the wall clock time and one core is never used.

Signed-off-by: Sidong Yang <sidong.yang@furiosa.ai>
---
 drivers/accel/rocket/rocket_drv.c | 60 +++++++++++++++++++++++++++++--
 drivers/accel/rocket/rocket_drv.h | 14 ++++++++
 drivers/accel/rocket/rocket_job.c |  7 ++++
 drivers/accel/rocket/rocket_job.h |  4 +++
 4 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c
index 8bbbce594883..5b81d1ecc0e4 100644
--- a/drivers/accel/rocket/rocket_drv.c
+++ b/drivers/accel/rocket/rocket_drv.c
@@ -3,8 +3,10 @@
 
 #include <drm/drm_accel.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_file.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_ioctl.h>
+#include <drm/drm_print.h>
 #include <drm/rocket_accel.h>
 #include <linux/clk.h>
 #include <linux/err.h>
@@ -68,6 +70,25 @@ rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
 	kref_put(&domain->kref, rocket_iommu_domain_destroy);
 }
 
+static void
+rocket_engine_usage_destroy(struct kref *kref)
+{
+	kfree(container_of(kref, struct rocket_engine_usage, kref));
+}
+
+struct rocket_engine_usage *
+rocket_engine_usage_get(struct rocket_file_priv *rocket_priv)
+{
+	kref_get(&rocket_priv->engine_usage->kref);
+	return rocket_priv->engine_usage;
+}
+
+void
+rocket_engine_usage_put(struct rocket_engine_usage *usage)
+{
+	kref_put(&usage->kref, rocket_engine_usage_destroy);
+}
+
 static int
 rocket_open(struct drm_device *dev, struct drm_file *file)
 {
@@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
 	}
 
 	rocket_priv->rdev = rdev;
+
+	rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
+						 elapsed_ns, rdev->num_cores);
+	if (!rocket_priv->engine_usage) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+	kref_init(&rocket_priv->engine_usage->kref);
+
 	rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
 	if (IS_ERR(rocket_priv->domain)) {
 		ret = PTR_ERR(rocket_priv->domain);
-		goto err_free;
+		goto err_put_engine_usage;
 	}
 
 	file->driver_priv = rocket_priv;
@@ -109,6 +139,8 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
 	mutex_destroy(&rocket_priv->mm_lock);
 	drm_mm_takedown(&rocket_priv->mm);
 	rocket_iommu_domain_put(rocket_priv->domain);
+err_put_engine_usage:
+	rocket_engine_usage_put(rocket_priv->engine_usage);
 err_free:
 	kfree(rocket_priv);
 err_put_mod:
@@ -125,6 +157,7 @@ rocket_postclose(struct drm_device *dev, struct drm_file *file)
 	mutex_destroy(&rocket_priv->mm_lock);
 	drm_mm_takedown(&rocket_priv->mm);
 	rocket_iommu_domain_put(rocket_priv->domain);
+	rocket_engine_usage_put(rocket_priv->engine_usage);
 	kfree(rocket_priv);
 	module_put(THIS_MODULE);
 }
@@ -139,7 +172,29 @@ static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
 	ROCKET_IOCTL(FINI_BO, fini_bo),
 };
 
-DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
+static void rocket_show_fdinfo(struct drm_printer *p, struct drm_file *file)
+{
+	struct rocket_device *rdev = to_rocket_device(file->minor->dev);
+	struct rocket_file_priv *rocket_priv = file->driver_priv;
+	unsigned int core;
+
+	/*
+	 * These are estimates: the time is taken around the whole period a job
+	 * occupies a core, so it also covers handling the completion interrupt
+	 * and, for a job made of several tasks, the gaps between them.
+	 */
+	for (core = 0; core < rdev->num_cores; core++)
+		drm_printf(p, "drm-engine-npu%u:\t%llu ns\n", core,
+			   rocket_priv->engine_usage->elapsed_ns[core]);
+
+	drm_show_memory_stats(p, file);
+}
+
+static const struct file_operations rocket_accel_driver_fops = {
+	.owner = THIS_MODULE,
+	DRM_ACCEL_FOPS,
+	.show_fdinfo = drm_show_fdinfo,
+};
 
 /*
  * Rocket driver version:
@@ -150,6 +205,7 @@ static const struct drm_driver rocket_drm_driver = {
 	.open			= rocket_open,
 	.postclose		= rocket_postclose,
 	.gem_create_object	= rocket_gem_create_object,
+	.show_fdinfo		= rocket_show_fdinfo,
 	.ioctls			= rocket_drm_driver_ioctls,
 	.num_ioctls		= ARRAY_SIZE(rocket_drm_driver_ioctls),
 	.fops			= &rocket_accel_driver_fops,
diff --git a/drivers/accel/rocket/rocket_drv.h b/drivers/accel/rocket/rocket_drv.h
index 2c673bb99ccc..eff5c2ac54d1 100644
--- a/drivers/accel/rocket/rocket_drv.h
+++ b/drivers/accel/rocket/rocket_drv.h
@@ -16,6 +16,15 @@ struct rocket_iommu_domain {
 	struct kref kref;
 };
 
+/*
+ * Time each core has spent running a client's jobs, in nanoseconds, indexed by
+ * core. Jobs hold a reference, as they can outlive the file that submitted them.
+ */
+struct rocket_engine_usage {
+	struct kref kref;
+	u64 elapsed_ns[];
+};
+
 struct rocket_file_priv {
 	struct rocket_device *rdev;
 
@@ -24,9 +33,14 @@ struct rocket_file_priv {
 	struct mutex mm_lock;
 
 	struct drm_sched_entity sched_entity;
+
+	struct rocket_engine_usage *engine_usage;
 };
 
 struct rocket_iommu_domain *rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv);
 void rocket_iommu_domain_put(struct rocket_iommu_domain *domain);
 
+struct rocket_engine_usage *rocket_engine_usage_get(struct rocket_file_priv *rocket_priv);
+void rocket_engine_usage_put(struct rocket_engine_usage *usage);
+
 #endif
diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
index ce994748dd5d..c0ffbccfb6cc 100644
--- a/drivers/accel/rocket/rocket_job.c
+++ b/drivers/accel/rocket/rocket_job.c
@@ -13,6 +13,7 @@
 #include <linux/iommu.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/sched/clock.h>
 
 #include "rocket_core.h"
 #include "rocket_device.h"
@@ -265,6 +266,7 @@ static void rocket_job_cleanup(struct kref *ref)
 	unsigned int i;
 
 	rocket_iommu_domain_put(job->domain);
+	rocket_engine_usage_put(job->engine_usage);
 
 	dma_fence_put(job->done_fence);
 	dma_fence_put(job->inference_done_fence);
@@ -351,6 +353,7 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
 
 	scoped_guard(mutex, &core->job_lock) {
 		core->in_flight_job = job;
+		job->start_ns = local_clock();
 		rocket_job_hw_submit(core, job);
 	}
 
@@ -379,6 +382,9 @@ static void rocket_job_handle_irq(struct rocket_core *core)
 				return;
 			}
 
+			core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
+				local_clock() - core->in_flight_job->start_ns;
+
 			iommu_detach_group(NULL, iommu_group_get(core->dev));
 			dma_fence_signal(core->in_flight_job->done_fence);
 			pm_runtime_put_autosuspend(core->dev);
@@ -587,6 +593,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 
 	rjob->rdev = rdev;
 	rjob->domain = rocket_iommu_domain_get(file_priv);
+	rjob->engine_usage = rocket_engine_usage_get(file_priv);
 
 	ret = drm_sched_job_init(&rjob->base,
 				 &file_priv->sched_entity,
diff --git a/drivers/accel/rocket/rocket_job.h b/drivers/accel/rocket/rocket_job.h
index 4ae00feec3b9..14aa78aebc99 100644
--- a/drivers/accel/rocket/rocket_job.h
+++ b/drivers/accel/rocket/rocket_job.h
@@ -38,6 +38,10 @@ struct rocket_job {
 
 	struct rocket_iommu_domain *domain;
 
+	/* When the job was handed to a core, and where to account the time */
+	u64 start_ns;
+	struct rocket_engine_usage *engine_usage;
+
 	struct kref refcount;
 };
 

base-commit: a1e63ab695ad76850ab9787b9eb6aab695dc2965
-- 
2.53.0

Re: [PATCH] accel/rocket: Report per-core engine usage through fdinfo

[Gyeyoung Baek]

Hi Sidong,

> If userspace (for example systemd-udevd reacting to the drm_dev_register()
> uevent, or a client opening the device while remaining cores are still probing
> or deferred) calls rocket_open() before all cores have finished probing,
> rocket_priv->engine_usage is allocated with the smaller rdev->num_cores count
> at that moment. Once the remaining cores finish probing and increment
> rdev->num_cores, calling rocket_show_fdinfo() will iterate up to the new
> rdev->num_cores and read past the end of elapsed_ns[].
>
> Similarly, if a lower-indexed core is unbound via rocket_remove(),
> rdev->num_cores is decremented while higher-indexed cores retain their
> original core->index. If a file is opened afterwards, rocket_job_handle_irq()
> indexing engine_usage->elapsed_ns[core->index] can also write out of bounds:
>

I reproduced the OOB access sashiko reported.
I used `driver_override` to keep core1 and core2 from
binding, so only core0 had been probed when the file was opened:

  BUG: KASAN: slab-out-of-bounds in rocket_show_fdinfo+0xac/0x120 [rocket]
  Read of size 8 at addr ffff00000242b230 by task grep/461

  Hardware name: Radxa ROCK 5B (DT)
  Call trace:
   rocket_show_fdinfo+0xac/0x120 [rocket]
   drm_show_fdinfo+0x218/0x2a8 [drm]
   seq_show+0x340/0x648
   seq_read_iter+0x318/0x9d8
   vfs_read+0x220/0x6a8
   ksys_read+0x108/0x1f8
   ...

  Allocated by task 444 on cpu 5 at 79.841431s:
   __kmalloc_noprof+0x2d0/0x740
   rocket_open+0xb4/0x468 [rocket]
   drm_file_alloc+0x37c/0x568 [drm]
   drm_open_helper+0x110/0x2a8 [drm]
   accel_open+0xc4/0x1c0 [drm]
   ...

> drivers/accel/rocket/rocket_job.c:rocket_job_handle_irq() {
>         ...
>         core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
>                 local_clock() - core->in_flight_job->start_ns;
>         ...
> }
>
> Would it make sense to record the total core count allocated for rdev->cores
> in rocket_device_init() (or store the allocated array length in struct
> rocket_engine_usage) so that the allocation size and bounds stay consistent?
>

`num_cores` is used with two different meanings:

  1. in rocket_device_init(): the number of core nodes in the DT
         (the size of the cores[] allocation)
  2. rdev->num_cores: the number of cores currently bound

This patch uses (2) where (1) is needed. How about storing (1) in a
new member (e.g. max_cores), as sashiko suggested, and using it instead?

> --
> Sashiko AI review · https://sashiko.dev/#/patchset/20261031083642.1076356-1-sidong.yang@furiosa.ai?part=1


--
Thanks,
Gyeyoung

Re: [PATCH] accel/rocket: Report per-core engine usage through fdinfo

[Sidong Yang]

On Sun, Nov 01, 2026 at 08:55:15PM +0900, Gyeyoung Baek wrote:
> Hi Sidong,
> 
> > If userspace (for example systemd-udevd reacting to the drm_dev_register()
> > uevent, or a client opening the device while remaining cores are still probing
> > or deferred) calls rocket_open() before all cores have finished probing,
> > rocket_priv->engine_usage is allocated with the smaller rdev->num_cores count
> > at that moment. Once the remaining cores finish probing and increment
> > rdev->num_cores, calling rocket_show_fdinfo() will iterate up to the new
> > rdev->num_cores and read past the end of elapsed_ns[].
> >
> > Similarly, if a lower-indexed core is unbound via rocket_remove(),
> > rdev->num_cores is decremented while higher-indexed cores retain their
> > original core->index. If a file is opened afterwards, rocket_job_handle_irq()
> > indexing engine_usage->elapsed_ns[core->index] can also write out of bounds:
> >
> 
> I reproduced the OOB access sashiko reported.
> I used `driver_override` to keep core1 and core2 from
> binding, so only core0 had been probed when the file was opened:
> 
>   BUG: KASAN: slab-out-of-bounds in rocket_show_fdinfo+0xac/0x120 [rocket]
>   Read of size 8 at addr ffff00000242b230 by task grep/461
> 
>   Hardware name: Radxa ROCK 5B (DT)
>   Call trace:
>    rocket_show_fdinfo+0xac/0x120 [rocket]
>    drm_show_fdinfo+0x218/0x2a8 [drm]
>    seq_show+0x340/0x648
>    seq_read_iter+0x318/0x9d8
>    vfs_read+0x220/0x6a8
>    ksys_read+0x108/0x1f8
>    ...
> 
>   Allocated by task 444 on cpu 5 at 79.841431s:
>    __kmalloc_noprof+0x2d0/0x740
>    rocket_open+0xb4/0x468 [rocket]
>    drm_file_alloc+0x37c/0x568 [drm]
>    drm_open_helper+0x110/0x2a8 [drm]
>    accel_open+0xc4/0x1c0 [drm]
>    ...
> 
> > drivers/accel/rocket/rocket_job.c:rocket_job_handle_irq() {
> >         ...
> >         core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
> >                 local_clock() - core->in_flight_job->start_ns;
> >         ...
> > }
> >
> > Would it make sense to record the total core count allocated for rdev->cores
> > in rocket_device_init() (or store the allocated array length in struct
> > rocket_engine_usage) so that the allocation size and bounds stay consistent?
> >
> 
> `num_cores` is used with two different meanings:
> 
>   1. in rocket_device_init(): the number of core nodes in the DT
>          (the size of the cores[] allocation)
>   2. rdev->num_cores: the number of cores currently bound
> 
> This patch uses (2) where (1) is needed. How about storing (1) in a
> new member (e.g. max_cores), as sashiko suggested, and using it instead?

Hi Gyeyoung,

Thanks for reproducing it. You're right.
I'll send v2 with max_cores as you suggested.

Thanks,
Sidong
> 
> > --
> > Sashiko AI review · https://sashiko.dev/#/patchset/20261031083642.1076356-1-sidong.yang@furiosa.ai?part=1
> 
> 
> --
> Thanks,
> Gyeyoung