[PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[Sidong Yang]

There is no way to tell how much of the NPU a client uses, or whether its
jobs reach more than one core. Implement the DRM usage stats: account the
time each job occupies a core and report it per client as
drm-engine-npu<n>, along with drm_show_memory_stats() for the BOs.

Jobs can outlive the file that submitted them, so the counters are
refcounted and each job holds a reference. The time covers the whole
period a job owns a core, so it also includes handling the completion
interrupt and the gaps between the tasks of a multi-task job.

The array is sized by rdev->max_cores, the number of core slots, not by
the number of cores bound when the file is opened. A core can still be
probing while a client opens the device, and the IRQ handler indexes the
array with core->index, which stays valid when a core is unbound. A slot
with no core bound reads as zero.

For example, teflon running a model two operations deep 3001 times gives:

  drm-engine-npu0:	22719385 ns
  drm-engine-npu1:	23345296 ns
  drm-engine-npu2:	0 ns

The NPU is busy 15.6% of the wall clock time and one core is never used.

Signed-off-by: Sidong Yang <sidong.yang@furiosa.ai>
---
v2: Size the per-core counters by rdev->max_cores, the number of core
    slots, instead of the number of cores bound when the file is opened.
    Sashiko reported the out-of-bounds read and Gyeyoung Baek reproduced
    it with KASAN.

    This depends on commit 6eff1d69dae8 ("accel/rocket: search every core
    slot when a core is removed"), which added rdev->max_cores. It is in
    mainline through drm-misc-fixes but not yet in drm-misc-next.

Link to v1: https://lore.kernel.org/all/20261031083642.1076356-1-sidong.yang@furiosa.ai/

 drivers/accel/rocket/rocket_drv.c | 60 +++++++++++++++++++++++++++++--
 drivers/accel/rocket/rocket_drv.h | 14 ++++++++
 drivers/accel/rocket/rocket_job.c |  7 ++++
 drivers/accel/rocket/rocket_job.h |  4 +++
 4 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c
index 2bcfe4ab3c68..58a486206565 100644
--- a/drivers/accel/rocket/rocket_drv.c
+++ b/drivers/accel/rocket/rocket_drv.c
@@ -3,8 +3,10 @@
 
 #include <drm/drm_accel.h>
 #include <drm/drm_drv.h>
+#include <drm/drm_file.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_ioctl.h>
+#include <drm/drm_print.h>
 #include <drm/rocket_accel.h>
 #include <linux/clk.h>
 #include <linux/err.h>
@@ -68,6 +70,25 @@ rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
 	kref_put(&domain->kref, rocket_iommu_domain_destroy);
 }
 
+static void
+rocket_engine_usage_destroy(struct kref *kref)
+{
+	kfree(container_of(kref, struct rocket_engine_usage, kref));
+}
+
+struct rocket_engine_usage *
+rocket_engine_usage_get(struct rocket_file_priv *rocket_priv)
+{
+	kref_get(&rocket_priv->engine_usage->kref);
+	return rocket_priv->engine_usage;
+}
+
+void
+rocket_engine_usage_put(struct rocket_engine_usage *usage)
+{
+	kref_put(&usage->kref, rocket_engine_usage_destroy);
+}
+
 static int
 rocket_open(struct drm_device *dev, struct drm_file *file)
 {
@@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
 	}
 
 	rocket_priv->rdev = rdev;
+
+	rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
+						 elapsed_ns, rdev->max_cores);
+	if (!rocket_priv->engine_usage) {
+		ret = -ENOMEM;
+		goto err_free;
+	}
+	kref_init(&rocket_priv->engine_usage->kref);
+
 	rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
 	if (IS_ERR(rocket_priv->domain)) {
 		ret = PTR_ERR(rocket_priv->domain);
-		goto err_free;
+		goto err_put_engine_usage;
 	}
 
 	file->driver_priv = rocket_priv;
@@ -109,6 +139,8 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
 	mutex_destroy(&rocket_priv->mm_lock);
 	drm_mm_takedown(&rocket_priv->mm);
 	rocket_iommu_domain_put(rocket_priv->domain);
+err_put_engine_usage:
+	rocket_engine_usage_put(rocket_priv->engine_usage);
 err_free:
 	kfree(rocket_priv);
 err_put_mod:
@@ -125,6 +157,7 @@ rocket_postclose(struct drm_device *dev, struct drm_file *file)
 	mutex_destroy(&rocket_priv->mm_lock);
 	drm_mm_takedown(&rocket_priv->mm);
 	rocket_iommu_domain_put(rocket_priv->domain);
+	rocket_engine_usage_put(rocket_priv->engine_usage);
 	kfree(rocket_priv);
 	module_put(THIS_MODULE);
 }
@@ -139,7 +172,29 @@ static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
 	ROCKET_IOCTL(FINI_BO, fini_bo),
 };
 
-DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
+static void rocket_show_fdinfo(struct drm_printer *p, struct drm_file *file)
+{
+	struct rocket_device *rdev = to_rocket_device(file->minor->dev);
+	struct rocket_file_priv *rocket_priv = file->driver_priv;
+	unsigned int core;
+
+	/*
+	 * These are estimates: the time is taken around the whole period a job
+	 * occupies a core, so it also covers handling the completion interrupt
+	 * and, for a job made of several tasks, the gaps between them.
+	 */
+	for (core = 0; core < rdev->max_cores; core++)
+		drm_printf(p, "drm-engine-npu%u:\t%llu ns\n", core,
+			   rocket_priv->engine_usage->elapsed_ns[core]);
+
+	drm_show_memory_stats(p, file);
+}
+
+static const struct file_operations rocket_accel_driver_fops = {
+	.owner = THIS_MODULE,
+	DRM_ACCEL_FOPS,
+	.show_fdinfo = drm_show_fdinfo,
+};
 
 /*
  * Rocket driver version:
@@ -150,6 +205,7 @@ static const struct drm_driver rocket_drm_driver = {
 	.open			= rocket_open,
 	.postclose		= rocket_postclose,
 	.gem_create_object	= rocket_gem_create_object,
+	.show_fdinfo		= rocket_show_fdinfo,
 	.ioctls			= rocket_drm_driver_ioctls,
 	.num_ioctls		= ARRAY_SIZE(rocket_drm_driver_ioctls),
 	.fops			= &rocket_accel_driver_fops,
diff --git a/drivers/accel/rocket/rocket_drv.h b/drivers/accel/rocket/rocket_drv.h
index 2c673bb99ccc..eff5c2ac54d1 100644
--- a/drivers/accel/rocket/rocket_drv.h
+++ b/drivers/accel/rocket/rocket_drv.h
@@ -16,6 +16,15 @@ struct rocket_iommu_domain {
 	struct kref kref;
 };
 
+/*
+ * Time each core has spent running a client's jobs, in nanoseconds, indexed by
+ * core. Jobs hold a reference, as they can outlive the file that submitted them.
+ */
+struct rocket_engine_usage {
+	struct kref kref;
+	u64 elapsed_ns[];
+};
+
 struct rocket_file_priv {
 	struct rocket_device *rdev;
 
@@ -24,9 +33,14 @@ struct rocket_file_priv {
 	struct mutex mm_lock;
 
 	struct drm_sched_entity sched_entity;
+
+	struct rocket_engine_usage *engine_usage;
 };
 
 struct rocket_iommu_domain *rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv);
 void rocket_iommu_domain_put(struct rocket_iommu_domain *domain);
 
+struct rocket_engine_usage *rocket_engine_usage_get(struct rocket_file_priv *rocket_priv);
+void rocket_engine_usage_put(struct rocket_engine_usage *usage);
+
 #endif
diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
index ce994748dd5d..c0ffbccfb6cc 100644
--- a/drivers/accel/rocket/rocket_job.c
+++ b/drivers/accel/rocket/rocket_job.c
@@ -13,6 +13,7 @@
 #include <linux/iommu.h>
 #include <linux/platform_device.h>
 #include <linux/pm_runtime.h>
+#include <linux/sched/clock.h>
 
 #include "rocket_core.h"
 #include "rocket_device.h"
@@ -265,6 +266,7 @@ static void rocket_job_cleanup(struct kref *ref)
 	unsigned int i;
 
 	rocket_iommu_domain_put(job->domain);
+	rocket_engine_usage_put(job->engine_usage);
 
 	dma_fence_put(job->done_fence);
 	dma_fence_put(job->inference_done_fence);
@@ -351,6 +353,7 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
 
 	scoped_guard(mutex, &core->job_lock) {
 		core->in_flight_job = job;
+		job->start_ns = local_clock();
 		rocket_job_hw_submit(core, job);
 	}
 
@@ -379,6 +382,9 @@ static void rocket_job_handle_irq(struct rocket_core *core)
 				return;
 			}
 
+			core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
+				local_clock() - core->in_flight_job->start_ns;
+
 			iommu_detach_group(NULL, iommu_group_get(core->dev));
 			dma_fence_signal(core->in_flight_job->done_fence);
 			pm_runtime_put_autosuspend(core->dev);
@@ -587,6 +593,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
 
 	rjob->rdev = rdev;
 	rjob->domain = rocket_iommu_domain_get(file_priv);
+	rjob->engine_usage = rocket_engine_usage_get(file_priv);
 
 	ret = drm_sched_job_init(&rjob->base,
 				 &file_priv->sched_entity,
diff --git a/drivers/accel/rocket/rocket_job.h b/drivers/accel/rocket/rocket_job.h
index 4ae00feec3b9..14aa78aebc99 100644
--- a/drivers/accel/rocket/rocket_job.h
+++ b/drivers/accel/rocket/rocket_job.h
@@ -38,6 +38,10 @@ struct rocket_job {
 
 	struct rocket_iommu_domain *domain;
 
+	/* When the job was handed to a core, and where to account the time */
+	u64 start_ns;
+	struct rocket_engine_usage *engine_usage;
+
 	struct kref refcount;
 };
 

base-commit: a1e63ab695ad76850ab9787b9eb6aab695dc2965
prerequisite-patch-id: 519bcdfdde80d902309c8346f749ebc4bb6b29c0
-- 
2.53.0

Re: [PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[Gyeyoung Baek]

Hi Sidong,

> +
>  static int
>  rocket_open(struct drm_device *dev, struct drm_file *file)
>  {
> @@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
>         }
>
>         rocket_priv->rdev = rdev;
> +
> +       rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
> +                                                elapsed_ns, rdev->max_cores);
> +       if (!rocket_priv->engine_usage) {
> +               ret = -ENOMEM;
> +               goto err_free;
> +       }
> +       kref_init(&rocket_priv->engine_usage->kref);

How about moving these lines into a rocket_engine_usage_create() helper, to pair
with rocket_engine_usage_destroy()?

Otherwise LGTM. Tested on RK3588 (ROCK 5B) with KASAN: the OOB from v1 no
longer reproduces, and drm-engine-npu* times are accounted as expected
while running benchmark_model with teflon.

Reviewed-by: Gyeyoung Baek <gye976@gmail.com>
Tested-by: Gyeyoung Baek <gye976@gmail.com>

-- 
Thanks,
Gyeyoung

Re: [PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[Sidong Yang]

On Tue, Nov 03, 2026 at 07:14:14PM +0900, Gyeyoung Baek wrote:
> Hi Sidong,
> 
> > +
> >  static int
> >  rocket_open(struct drm_device *dev, struct drm_file *file)
> >  {
> > @@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
> >         }
> >
> >         rocket_priv->rdev = rdev;
> > +
> > +       rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
> > +                                                elapsed_ns, rdev->max_cores);
> > +       if (!rocket_priv->engine_usage) {
> > +               ret = -ENOMEM;
> > +               goto err_free;
> > +       }
> > +       kref_init(&rocket_priv->engine_usage->kref);
> 
> How about moving these lines into a rocket_engine_usage_create() helper, to pair
> with rocket_engine_usage_destroy()?

Sure, I'll make it for v3.

Thanks,
Sidong
> 
> Otherwise LGTM. Tested on RK3588 (ROCK 5B) with KASAN: the OOB from v1 no
> longer reproduces, and drm-engine-npu* times are accounted as expected
> while running benchmark_model with teflon.
> 
> Reviewed-by: Gyeyoung Baek <gye976@gmail.com>
> Tested-by: Gyeyoung Baek <gye976@gmail.com>
> 
> -- 
> Thanks,
> Gyeyoung

Re: [PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[Tomeu Vizoso]

Hi Sidong,

can you please check the drm usage stats spec at
Documentation/gpu/drm-usage-stats.rst ?

I think you need to conform to the key naming part of it, with a
single key with a capacity equal to the number of cores. We also need
an entry in Documentation/gpu/drm-usage-stats.rst like the other
drivers have.

Btw, with a perfmon ioctl, we could have Perfetto integration and
benefit from other work such as drm-sched tracepoint visualization.
Just saying in case it can help in your use cases.

Thanks,

Tomeu

On Sun, Nov 1, 2026 at 4:27 PM Sidong Yang <sidong.yang@furiosa.ai> wrote:
>
> There is no way to tell how much of the NPU a client uses, or whether its
> jobs reach more than one core. Implement the DRM usage stats: account the
> time each job occupies a core and report it per client as
> drm-engine-npu<n>, along with drm_show_memory_stats() for the BOs.
>
> Jobs can outlive the file that submitted them, so the counters are
> refcounted and each job holds a reference. The time covers the whole
> period a job owns a core, so it also includes handling the completion
> interrupt and the gaps between the tasks of a multi-task job.
>
> The array is sized by rdev->max_cores, the number of core slots, not by
> the number of cores bound when the file is opened. A core can still be
> probing while a client opens the device, and the IRQ handler indexes the
> array with core->index, which stays valid when a core is unbound. A slot
> with no core bound reads as zero.
>
> For example, teflon running a model two operations deep 3001 times gives:
>
>   drm-engine-npu0:      22719385 ns
>   drm-engine-npu1:      23345296 ns
>   drm-engine-npu2:      0 ns
>
> The NPU is busy 15.6% of the wall clock time and one core is never used.
>
> Signed-off-by: Sidong Yang <sidong.yang@furiosa.ai>
> ---
> v2: Size the per-core counters by rdev->max_cores, the number of core
>     slots, instead of the number of cores bound when the file is opened.
>     Sashiko reported the out-of-bounds read and Gyeyoung Baek reproduced
>     it with KASAN.
>
>     This depends on commit 6eff1d69dae8 ("accel/rocket: search every core
>     slot when a core is removed"), which added rdev->max_cores. It is in
>     mainline through drm-misc-fixes but not yet in drm-misc-next.
>
> Link to v1: https://lore.kernel.org/all/20261031083642.1076356-1-sidong.yang@furiosa.ai/
>
>  drivers/accel/rocket/rocket_drv.c | 60 +++++++++++++++++++++++++++++--
>  drivers/accel/rocket/rocket_drv.h | 14 ++++++++
>  drivers/accel/rocket/rocket_job.c |  7 ++++
>  drivers/accel/rocket/rocket_job.h |  4 +++
>  4 files changed, 83 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c
> index 2bcfe4ab3c68..58a486206565 100644
> --- a/drivers/accel/rocket/rocket_drv.c
> +++ b/drivers/accel/rocket/rocket_drv.c
> @@ -3,8 +3,10 @@
>
>  #include <drm/drm_accel.h>
>  #include <drm/drm_drv.h>
> +#include <drm/drm_file.h>
>  #include <drm/drm_gem.h>
>  #include <drm/drm_ioctl.h>
> +#include <drm/drm_print.h>
>  #include <drm/rocket_accel.h>
>  #include <linux/clk.h>
>  #include <linux/err.h>
> @@ -68,6 +70,25 @@ rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
>         kref_put(&domain->kref, rocket_iommu_domain_destroy);
>  }
>
> +static void
> +rocket_engine_usage_destroy(struct kref *kref)
> +{
> +       kfree(container_of(kref, struct rocket_engine_usage, kref));
> +}
> +
> +struct rocket_engine_usage *
> +rocket_engine_usage_get(struct rocket_file_priv *rocket_priv)
> +{
> +       kref_get(&rocket_priv->engine_usage->kref);
> +       return rocket_priv->engine_usage;
> +}
> +
> +void
> +rocket_engine_usage_put(struct rocket_engine_usage *usage)
> +{
> +       kref_put(&usage->kref, rocket_engine_usage_destroy);
> +}
> +
>  static int
>  rocket_open(struct drm_device *dev, struct drm_file *file)
>  {
> @@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
>         }
>
>         rocket_priv->rdev = rdev;
> +
> +       rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
> +                                                elapsed_ns, rdev->max_cores);
> +       if (!rocket_priv->engine_usage) {
> +               ret = -ENOMEM;
> +               goto err_free;
> +       }
> +       kref_init(&rocket_priv->engine_usage->kref);
> +
>         rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
>         if (IS_ERR(rocket_priv->domain)) {
>                 ret = PTR_ERR(rocket_priv->domain);
> -               goto err_free;
> +               goto err_put_engine_usage;
>         }
>
>         file->driver_priv = rocket_priv;
> @@ -109,6 +139,8 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
>         mutex_destroy(&rocket_priv->mm_lock);
>         drm_mm_takedown(&rocket_priv->mm);
>         rocket_iommu_domain_put(rocket_priv->domain);
> +err_put_engine_usage:
> +       rocket_engine_usage_put(rocket_priv->engine_usage);
>  err_free:
>         kfree(rocket_priv);
>  err_put_mod:
> @@ -125,6 +157,7 @@ rocket_postclose(struct drm_device *dev, struct drm_file *file)
>         mutex_destroy(&rocket_priv->mm_lock);
>         drm_mm_takedown(&rocket_priv->mm);
>         rocket_iommu_domain_put(rocket_priv->domain);
> +       rocket_engine_usage_put(rocket_priv->engine_usage);
>         kfree(rocket_priv);
>         module_put(THIS_MODULE);
>  }
> @@ -139,7 +172,29 @@ static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
>         ROCKET_IOCTL(FINI_BO, fini_bo),
>  };
>
> -DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
> +static void rocket_show_fdinfo(struct drm_printer *p, struct drm_file *file)
> +{
> +       struct rocket_device *rdev = to_rocket_device(file->minor->dev);
> +       struct rocket_file_priv *rocket_priv = file->driver_priv;
> +       unsigned int core;
> +
> +       /*
> +        * These are estimates: the time is taken around the whole period a job
> +        * occupies a core, so it also covers handling the completion interrupt
> +        * and, for a job made of several tasks, the gaps between them.
> +        */
> +       for (core = 0; core < rdev->max_cores; core++)
> +               drm_printf(p, "drm-engine-npu%u:\t%llu ns\n", core,
> +                          rocket_priv->engine_usage->elapsed_ns[core]);
> +
> +       drm_show_memory_stats(p, file);
> +}
> +
> +static const struct file_operations rocket_accel_driver_fops = {
> +       .owner = THIS_MODULE,
> +       DRM_ACCEL_FOPS,
> +       .show_fdinfo = drm_show_fdinfo,
> +};
>
>  /*
>   * Rocket driver version:
> @@ -150,6 +205,7 @@ static const struct drm_driver rocket_drm_driver = {
>         .open                   = rocket_open,
>         .postclose              = rocket_postclose,
>         .gem_create_object      = rocket_gem_create_object,
> +       .show_fdinfo            = rocket_show_fdinfo,
>         .ioctls                 = rocket_drm_driver_ioctls,
>         .num_ioctls             = ARRAY_SIZE(rocket_drm_driver_ioctls),
>         .fops                   = &rocket_accel_driver_fops,
> diff --git a/drivers/accel/rocket/rocket_drv.h b/drivers/accel/rocket/rocket_drv.h
> index 2c673bb99ccc..eff5c2ac54d1 100644
> --- a/drivers/accel/rocket/rocket_drv.h
> +++ b/drivers/accel/rocket/rocket_drv.h
> @@ -16,6 +16,15 @@ struct rocket_iommu_domain {
>         struct kref kref;
>  };
>
> +/*
> + * Time each core has spent running a client's jobs, in nanoseconds, indexed by
> + * core. Jobs hold a reference, as they can outlive the file that submitted them.
> + */
> +struct rocket_engine_usage {
> +       struct kref kref;
> +       u64 elapsed_ns[];
> +};
> +
>  struct rocket_file_priv {
>         struct rocket_device *rdev;
>
> @@ -24,9 +33,14 @@ struct rocket_file_priv {
>         struct mutex mm_lock;
>
>         struct drm_sched_entity sched_entity;
> +
> +       struct rocket_engine_usage *engine_usage;
>  };
>
>  struct rocket_iommu_domain *rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv);
>  void rocket_iommu_domain_put(struct rocket_iommu_domain *domain);
>
> +struct rocket_engine_usage *rocket_engine_usage_get(struct rocket_file_priv *rocket_priv);
> +void rocket_engine_usage_put(struct rocket_engine_usage *usage);
> +
>  #endif
> diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
> index ce994748dd5d..c0ffbccfb6cc 100644
> --- a/drivers/accel/rocket/rocket_job.c
> +++ b/drivers/accel/rocket/rocket_job.c
> @@ -13,6 +13,7 @@
>  #include <linux/iommu.h>
>  #include <linux/platform_device.h>
>  #include <linux/pm_runtime.h>
> +#include <linux/sched/clock.h>
>
>  #include "rocket_core.h"
>  #include "rocket_device.h"
> @@ -265,6 +266,7 @@ static void rocket_job_cleanup(struct kref *ref)
>         unsigned int i;
>
>         rocket_iommu_domain_put(job->domain);
> +       rocket_engine_usage_put(job->engine_usage);
>
>         dma_fence_put(job->done_fence);
>         dma_fence_put(job->inference_done_fence);
> @@ -351,6 +353,7 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
>
>         scoped_guard(mutex, &core->job_lock) {
>                 core->in_flight_job = job;
> +               job->start_ns = local_clock();
>                 rocket_job_hw_submit(core, job);
>         }
>
> @@ -379,6 +382,9 @@ static void rocket_job_handle_irq(struct rocket_core *core)
>                                 return;
>                         }
>
> +                       core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
> +                               local_clock() - core->in_flight_job->start_ns;
> +
>                         iommu_detach_group(NULL, iommu_group_get(core->dev));
>                         dma_fence_signal(core->in_flight_job->done_fence);
>                         pm_runtime_put_autosuspend(core->dev);
> @@ -587,6 +593,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
>
>         rjob->rdev = rdev;
>         rjob->domain = rocket_iommu_domain_get(file_priv);
> +       rjob->engine_usage = rocket_engine_usage_get(file_priv);
>
>         ret = drm_sched_job_init(&rjob->base,
>                                  &file_priv->sched_entity,
> diff --git a/drivers/accel/rocket/rocket_job.h b/drivers/accel/rocket/rocket_job.h
> index 4ae00feec3b9..14aa78aebc99 100644
> --- a/drivers/accel/rocket/rocket_job.h
> +++ b/drivers/accel/rocket/rocket_job.h
> @@ -38,6 +38,10 @@ struct rocket_job {
>
>         struct rocket_iommu_domain *domain;
>
> +       /* When the job was handed to a core, and where to account the time */
> +       u64 start_ns;
> +       struct rocket_engine_usage *engine_usage;
> +
>         struct kref refcount;
>  };
>
>
> base-commit: a1e63ab695ad76850ab9787b9eb6aab695dc2965
> prerequisite-patch-id: 519bcdfdde80d902309c8346f749ebc4bb6b29c0
> --
> 2.53.0
>

Re: [PATCH v2] accel/rocket: Report per-core engine usage through fdinfo

[Sidong Yang]

On Tue, Nov 03, 2026 at 11:51:16AM +0100, Tomeu Vizoso wrote:
> Hi Sidong,
> 
> can you please check the drm usage stats spec at
> Documentation/gpu/drm-usage-stats.rst ?
> 
> I think you need to conform to the key naming part of it, with a
> single key with a capacity equal to the number of cores. We also need
> an entry in Documentation/gpu/drm-usage-stats.rst like the other
> drivers have.

Hi Tomeu,

Thanks, I'll send v4 with a single key and capacity, and the
documentation. v3 crossed with your mail, so please ignore it.

Thanks,
Sidong

> 
> Btw, with a perfmon ioctl, we could have Perfetto integration and
> benefit from other work such as drm-sched tracepoint visualization.
> Just saying in case it can help in your use cases.
> 
> Thanks,
> 
> Tomeu
> 
> On Sun, Nov 1, 2026 at 4:27 PM Sidong Yang <sidong.yang@furiosa.ai> wrote:
> >
> > There is no way to tell how much of the NPU a client uses, or whether its
> > jobs reach more than one core. Implement the DRM usage stats: account the
> > time each job occupies a core and report it per client as
> > drm-engine-npu<n>, along with drm_show_memory_stats() for the BOs.
> >
> > Jobs can outlive the file that submitted them, so the counters are
> > refcounted and each job holds a reference. The time covers the whole
> > period a job owns a core, so it also includes handling the completion
> > interrupt and the gaps between the tasks of a multi-task job.
> >
> > The array is sized by rdev->max_cores, the number of core slots, not by
> > the number of cores bound when the file is opened. A core can still be
> > probing while a client opens the device, and the IRQ handler indexes the
> > array with core->index, which stays valid when a core is unbound. A slot
> > with no core bound reads as zero.
> >
> > For example, teflon running a model two operations deep 3001 times gives:
> >
> >   drm-engine-npu0:      22719385 ns
> >   drm-engine-npu1:      23345296 ns
> >   drm-engine-npu2:      0 ns
> >
> > The NPU is busy 15.6% of the wall clock time and one core is never used.
> >
> > Signed-off-by: Sidong Yang <sidong.yang@furiosa.ai>
> > ---
> > v2: Size the per-core counters by rdev->max_cores, the number of core
> >     slots, instead of the number of cores bound when the file is opened.
> >     Sashiko reported the out-of-bounds read and Gyeyoung Baek reproduced
> >     it with KASAN.
> >
> >     This depends on commit 6eff1d69dae8 ("accel/rocket: search every core
> >     slot when a core is removed"), which added rdev->max_cores. It is in
> >     mainline through drm-misc-fixes but not yet in drm-misc-next.
> >
> > Link to v1: https://lore.kernel.org/all/20261031083642.1076356-1-sidong.yang@furiosa.ai/
> >
> >  drivers/accel/rocket/rocket_drv.c | 60 +++++++++++++++++++++++++++++--
> >  drivers/accel/rocket/rocket_drv.h | 14 ++++++++
> >  drivers/accel/rocket/rocket_job.c |  7 ++++
> >  drivers/accel/rocket/rocket_job.h |  4 +++
> >  4 files changed, 83 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/accel/rocket/rocket_drv.c b/drivers/accel/rocket/rocket_drv.c
> > index 2bcfe4ab3c68..58a486206565 100644
> > --- a/drivers/accel/rocket/rocket_drv.c
> > +++ b/drivers/accel/rocket/rocket_drv.c
> > @@ -3,8 +3,10 @@
> >
> >  #include <drm/drm_accel.h>
> >  #include <drm/drm_drv.h>
> > +#include <drm/drm_file.h>
> >  #include <drm/drm_gem.h>
> >  #include <drm/drm_ioctl.h>
> > +#include <drm/drm_print.h>
> >  #include <drm/rocket_accel.h>
> >  #include <linux/clk.h>
> >  #include <linux/err.h>
> > @@ -68,6 +70,25 @@ rocket_iommu_domain_put(struct rocket_iommu_domain *domain)
> >         kref_put(&domain->kref, rocket_iommu_domain_destroy);
> >  }
> >
> > +static void
> > +rocket_engine_usage_destroy(struct kref *kref)
> > +{
> > +       kfree(container_of(kref, struct rocket_engine_usage, kref));
> > +}
> > +
> > +struct rocket_engine_usage *
> > +rocket_engine_usage_get(struct rocket_file_priv *rocket_priv)
> > +{
> > +       kref_get(&rocket_priv->engine_usage->kref);
> > +       return rocket_priv->engine_usage;
> > +}
> > +
> > +void
> > +rocket_engine_usage_put(struct rocket_engine_usage *usage)
> > +{
> > +       kref_put(&usage->kref, rocket_engine_usage_destroy);
> > +}
> > +
> >  static int
> >  rocket_open(struct drm_device *dev, struct drm_file *file)
> >  {
> > @@ -86,10 +107,19 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
> >         }
> >
> >         rocket_priv->rdev = rdev;
> > +
> > +       rocket_priv->engine_usage = kzalloc_flex(*rocket_priv->engine_usage,
> > +                                                elapsed_ns, rdev->max_cores);
> > +       if (!rocket_priv->engine_usage) {
> > +               ret = -ENOMEM;
> > +               goto err_free;
> > +       }
> > +       kref_init(&rocket_priv->engine_usage->kref);
> > +
> >         rocket_priv->domain = rocket_iommu_domain_create(rdev->cores[0].dev);
> >         if (IS_ERR(rocket_priv->domain)) {
> >                 ret = PTR_ERR(rocket_priv->domain);
> > -               goto err_free;
> > +               goto err_put_engine_usage;
> >         }
> >
> >         file->driver_priv = rocket_priv;
> > @@ -109,6 +139,8 @@ rocket_open(struct drm_device *dev, struct drm_file *file)
> >         mutex_destroy(&rocket_priv->mm_lock);
> >         drm_mm_takedown(&rocket_priv->mm);
> >         rocket_iommu_domain_put(rocket_priv->domain);
> > +err_put_engine_usage:
> > +       rocket_engine_usage_put(rocket_priv->engine_usage);
> >  err_free:
> >         kfree(rocket_priv);
> >  err_put_mod:
> > @@ -125,6 +157,7 @@ rocket_postclose(struct drm_device *dev, struct drm_file *file)
> >         mutex_destroy(&rocket_priv->mm_lock);
> >         drm_mm_takedown(&rocket_priv->mm);
> >         rocket_iommu_domain_put(rocket_priv->domain);
> > +       rocket_engine_usage_put(rocket_priv->engine_usage);
> >         kfree(rocket_priv);
> >         module_put(THIS_MODULE);
> >  }
> > @@ -139,7 +172,29 @@ static const struct drm_ioctl_desc rocket_drm_driver_ioctls[] = {
> >         ROCKET_IOCTL(FINI_BO, fini_bo),
> >  };
> >
> > -DEFINE_DRM_ACCEL_FOPS(rocket_accel_driver_fops);
> > +static void rocket_show_fdinfo(struct drm_printer *p, struct drm_file *file)
> > +{
> > +       struct rocket_device *rdev = to_rocket_device(file->minor->dev);
> > +       struct rocket_file_priv *rocket_priv = file->driver_priv;
> > +       unsigned int core;
> > +
> > +       /*
> > +        * These are estimates: the time is taken around the whole period a job
> > +        * occupies a core, so it also covers handling the completion interrupt
> > +        * and, for a job made of several tasks, the gaps between them.
> > +        */
> > +       for (core = 0; core < rdev->max_cores; core++)
> > +               drm_printf(p, "drm-engine-npu%u:\t%llu ns\n", core,
> > +                          rocket_priv->engine_usage->elapsed_ns[core]);
> > +
> > +       drm_show_memory_stats(p, file);
> > +}
> > +
> > +static const struct file_operations rocket_accel_driver_fops = {
> > +       .owner = THIS_MODULE,
> > +       DRM_ACCEL_FOPS,
> > +       .show_fdinfo = drm_show_fdinfo,
> > +};
> >
> >  /*
> >   * Rocket driver version:
> > @@ -150,6 +205,7 @@ static const struct drm_driver rocket_drm_driver = {
> >         .open                   = rocket_open,
> >         .postclose              = rocket_postclose,
> >         .gem_create_object      = rocket_gem_create_object,
> > +       .show_fdinfo            = rocket_show_fdinfo,
> >         .ioctls                 = rocket_drm_driver_ioctls,
> >         .num_ioctls             = ARRAY_SIZE(rocket_drm_driver_ioctls),
> >         .fops                   = &rocket_accel_driver_fops,
> > diff --git a/drivers/accel/rocket/rocket_drv.h b/drivers/accel/rocket/rocket_drv.h
> > index 2c673bb99ccc..eff5c2ac54d1 100644
> > --- a/drivers/accel/rocket/rocket_drv.h
> > +++ b/drivers/accel/rocket/rocket_drv.h
> > @@ -16,6 +16,15 @@ struct rocket_iommu_domain {
> >         struct kref kref;
> >  };
> >
> > +/*
> > + * Time each core has spent running a client's jobs, in nanoseconds, indexed by
> > + * core. Jobs hold a reference, as they can outlive the file that submitted them.
> > + */
> > +struct rocket_engine_usage {
> > +       struct kref kref;
> > +       u64 elapsed_ns[];
> > +};
> > +
> >  struct rocket_file_priv {
> >         struct rocket_device *rdev;
> >
> > @@ -24,9 +33,14 @@ struct rocket_file_priv {
> >         struct mutex mm_lock;
> >
> >         struct drm_sched_entity sched_entity;
> > +
> > +       struct rocket_engine_usage *engine_usage;
> >  };
> >
> >  struct rocket_iommu_domain *rocket_iommu_domain_get(struct rocket_file_priv *rocket_priv);
> >  void rocket_iommu_domain_put(struct rocket_iommu_domain *domain);
> >
> > +struct rocket_engine_usage *rocket_engine_usage_get(struct rocket_file_priv *rocket_priv);
> > +void rocket_engine_usage_put(struct rocket_engine_usage *usage);
> > +
> >  #endif
> > diff --git a/drivers/accel/rocket/rocket_job.c b/drivers/accel/rocket/rocket_job.c
> > index ce994748dd5d..c0ffbccfb6cc 100644
> > --- a/drivers/accel/rocket/rocket_job.c
> > +++ b/drivers/accel/rocket/rocket_job.c
> > @@ -13,6 +13,7 @@
> >  #include <linux/iommu.h>
> >  #include <linux/platform_device.h>
> >  #include <linux/pm_runtime.h>
> > +#include <linux/sched/clock.h>
> >
> >  #include "rocket_core.h"
> >  #include "rocket_device.h"
> > @@ -265,6 +266,7 @@ static void rocket_job_cleanup(struct kref *ref)
> >         unsigned int i;
> >
> >         rocket_iommu_domain_put(job->domain);
> > +       rocket_engine_usage_put(job->engine_usage);
> >
> >         dma_fence_put(job->done_fence);
> >         dma_fence_put(job->inference_done_fence);
> > @@ -351,6 +353,7 @@ static struct dma_fence *rocket_job_run(struct drm_sched_job *sched_job)
> >
> >         scoped_guard(mutex, &core->job_lock) {
> >                 core->in_flight_job = job;
> > +               job->start_ns = local_clock();
> >                 rocket_job_hw_submit(core, job);
> >         }
> >
> > @@ -379,6 +382,9 @@ static void rocket_job_handle_irq(struct rocket_core *core)
> >                                 return;
> >                         }
> >
> > +                       core->in_flight_job->engine_usage->elapsed_ns[core->index] +=
> > +                               local_clock() - core->in_flight_job->start_ns;
> > +
> >                         iommu_detach_group(NULL, iommu_group_get(core->dev));
> >                         dma_fence_signal(core->in_flight_job->done_fence);
> >                         pm_runtime_put_autosuspend(core->dev);
> > @@ -587,6 +593,7 @@ static int rocket_ioctl_submit_job(struct drm_device *dev, struct drm_file *file
> >
> >         rjob->rdev = rdev;
> >         rjob->domain = rocket_iommu_domain_get(file_priv);
> > +       rjob->engine_usage = rocket_engine_usage_get(file_priv);
> >
> >         ret = drm_sched_job_init(&rjob->base,
> >                                  &file_priv->sched_entity,
> > diff --git a/drivers/accel/rocket/rocket_job.h b/drivers/accel/rocket/rocket_job.h
> > index 4ae00feec3b9..14aa78aebc99 100644
> > --- a/drivers/accel/rocket/rocket_job.h
> > +++ b/drivers/accel/rocket/rocket_job.h
> > @@ -38,6 +38,10 @@ struct rocket_job {
> >
> >         struct rocket_iommu_domain *domain;
> >
> > +       /* When the job was handed to a core, and where to account the time */
> > +       u64 start_ns;
> > +       struct rocket_engine_usage *engine_usage;
> > +
> >         struct kref refcount;
> >  };
> >
> >
> > base-commit: a1e63ab695ad76850ab9787b9eb6aab695dc2965
> > prerequisite-patch-id: 519bcdfdde80d902309c8346f749ebc4bb6b29c0
> > --
> > 2.53.0
> >