[PATCH v3 1/5] drm/amdgpu: fix NULL deref in error path

[PATCH v3 1/5] drm/amdgpu: fix NULL deref in error path

[Pierre-Eric Pelloux-Prayer]

IF we hit the "error_free_root" error path, amdgpu_vm_pt_free_root
is called which will clear vm->root.bo so calling amdgpu_bo_unreserve
will crash.
Since root_bo is a reference to the BO previously installed as
vm->root.bo, use it.

Fixes: 8473bfdcb5b1 ("drm/amdgpu: fix error handling in amdgpu_vm_init")
Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 000c982816a7..ea8688061237 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -2716,7 +2716,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 error_free_root:
 	amdgpu_vm_pt_free_root(adev, vm);
-	amdgpu_bo_unreserve(vm->root.bo);
+	amdgpu_bo_unreserve(root_bo);
 	amdgpu_bo_unref(&root_bo);
 
 error_free_delayed:
-- 
2.43.0

[PATCH v3 2/5] drm/amdgpu: add gpu_recovery_enabled flag to amdgpu_device

[Pierre-Eric Pelloux-Prayer]

Add a per-device boolean to control whether GPU recovery is attempted
on a hang, independently of the global amdgpu.gpu_recovery module
parameter. It defaults to true and is exposed as a write to the
existing amdgpu_gpu_recover debugfs file, so a single device can have
auto-recovery disabled without affecting every other GPU in the
system.

amdgpu_device_should_recover_gpu() now takes this flag into account.

Assisted-by: LLM
Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  7 +++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  5 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 21 +++++++++++++++++----
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 5a32b54b1e9b..34bbb30b6f0c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -982,6 +982,13 @@ struct amdgpu_device {
 
 	struct amdgpu_uma_carveout_info uma_info;
 
+	/* Whether this device can attempt GPU recovery on a hang.
+	 * Defaults to true; can be turned off per-device (e.g. via the
+	 * amdgpu_gpu_recover debugfs file) independently of the global
+	 * amdgpu.gpu_recovery module parameter.
+	 */
+	bool gpu_recovery_enabled;
+
 	/* KFD
 	 * Must be last --ends in a flexible-array member.
 	 */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 5069042b5279..ec2514079919 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4109,6 +4109,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 		}
 	}
 
+	adev->gpu_recovery_enabled = true;
+
 fence_driver_init:
 	/* Fence driver */
 	r = amdgpu_fence_driver_sw_init(adev);
@@ -4908,6 +4910,9 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
 	if (amdgpu_gpu_recovery == 0)
 		goto disabled;
 
+	if (!adev->gpu_recovery_enabled)
+		goto disabled;
+
 	/* Skip soft reset check in fatal error mode */
 	if (!amdgpu_ras_is_poison_mode_supported(adev))
 		return true;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index bff2d38f3571..c54ff34572b0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -980,9 +980,13 @@ static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
 }
 
 /*
- * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
+ * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover,
+ * and control whether this device is allowed to auto-recover from a hang.
  *
- * Manually trigger a gpu reset at the next fence wait.
+ * Read triggers a gpu reset at the next fence wait.
+ *
+ * Write 0/1 to disable/enable auto GPU recovery for this device
+ * (equivalent to amdgpu.gpu_recovery=0, but scoped to this device only).
  */
 static int gpu_recover_get(void *data, u64 *val)
 {
@@ -1006,8 +1010,17 @@ static int gpu_recover_get(void *data, u64 *val)
 	return 0;
 }
 
+static int gpu_recover_set(void *data, u64 val)
+{
+	struct amdgpu_device *adev = (struct amdgpu_device *)data;
+
+	adev->gpu_recovery_enabled = !!val;
+
+	return 0;
+}
+
 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
-DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
+DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, gpu_recover_set,
 			 "%lld\n");
 
 static void amdgpu_debugfs_reset_work(struct work_struct *work)
@@ -1042,7 +1055,7 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
 	if (!amdgpu_sriov_vf(adev)) {
 
 		INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
-		debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
+		debugfs_create_file("amdgpu_gpu_recover", 0644, root, adev,
 				    &amdgpu_debugfs_gpu_recover_fops);
 	}
 #endif
-- 
2.43.0

[PATCH v3 3/5] drm/amdgpu: honor gpu_recovery_enabled in amdgpu_job_timedout

[Pierre-Eric Pelloux-Prayer]

The soft-recovery, per-ring-reset and IP-block-reset attempts in
amdgpu_job_timedout() were only gated on the global amdgpu_gpu_recovery
module parameter. Introduce a single can_reset variable that also
takes the per-device adev->gpu_recovery_enabled flag into account, and
use it for all three attempts instead of reading amdgpu_gpu_recovery
directly.

Assisted-by: LLM
Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 4d363bcfb6a5..0b8d0a79dd45 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -92,6 +92,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	struct drm_wedge_task_info *info = NULL;
 	struct amdgpu_task_info *ti = NULL;
 	struct amdgpu_device *adev = ring->adev;
+	bool can_reset = amdgpu_gpu_recovery && adev->gpu_recovery_enabled;
 	int idx, r;
 
 	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
@@ -111,7 +112,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	if (!amdgpu_sriov_vf(adev))
 		amdgpu_job_core_dump(adev, job);
 
-	if (amdgpu_gpu_recovery &&
+	if (can_reset &&
 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_SOFT_RECOVERY) &&
 	    amdgpu_ring_soft_recovery(ring, job->vmid, s_job->s_fence->parent)) {
 		dev_err(adev->dev, "ring %s timeout, but soft recovered\n",
@@ -130,7 +131,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	}
 
 	/* attempt a per ring reset */
-	if (amdgpu_gpu_recovery &&
+	if (can_reset &&
 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_PER_QUEUE) &&
 	    ring->funcs->reset) {
 		dev_err(adev->dev, "Starting %s ring reset\n",
@@ -152,7 +153,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	}
 
 	/* Attempt an IP block soft reset, if supported. */
-	if (amdgpu_gpu_recovery &&
+	if (can_reset &&
 	    amdgpu_ring_is_reset_type_supported(ring, AMDGPU_RESET_TYPE_IP_BLOCK_SOFT_RESET)) {
 		r = amdgpu_device_ip_soft_reset(ring, job->hw_fence);
 		if (!r) {
-- 
2.43.0

[PATCH v3 4/5] drm/amdgpu: add wedge event implementation

[Pierre-Eric Pelloux-Prayer]

When a job times out the kernel tries to reset the GPU to get
the system back to a normal state.
Sometimes it's useful to disable this process - for instance to
be able to inspect the hardware state at the time of the hang.
Since leaving fence unsignalled might affect the system's
stability, this commit enables the drm wedge framework for amdgpu.

When a hang condition is detected, the GPU isn't reset but all
the pending fences are signalled and the hung GPU cannot receive
new work. For KFD it's implemented by making
kfd_process_device_data_by_id return NULL if the requested GPU
is wedged.

When the user is done, it's possible to trigger a GPU
reset.

The auto-recovery is still enabled by default, and to use this
feature one has to either:
* boot with amdgpu.gpu_recovery=0: not recommended as it disables
  recovery for all GPUs
* write 0 to /sys/kernel/debug/dri/X/amdgpu_gpu_recover to disable
  auto-recovery for one GPU only

Runtime power management is disabled when the device is wedged
(because we can't submit any work to the GPU).

Assisted-by: LLM
Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 13 +++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  3 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  8 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  8 +++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    |  3 ++
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 34 ++++++++++++++++++++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 12 ++++++++
 drivers/gpu/drm/amd/amdkfd/kfd_process.c   |  6 +++-
 8 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index 34bbb30b6f0c..cd4959486c94 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -989,6 +989,13 @@ struct amdgpu_device {
 	 */
 	bool gpu_recovery_enabled;
 
+	/* Set to 1 when the device has been declared wedged following a
+	 * hang: it no longer accepts new work and pending fences have been
+	 * force-signalled. Reset to 0 once the device is reset back to a
+	 * working state.
+	 */
+	atomic_t wedge_status;
+
 	/* KFD
 	 * Must be last --ends in a flexible-array member.
 	 */
@@ -1374,4 +1381,10 @@ void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
 			   uint64_t uid);
 uint64_t amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
 			       enum amdgpu_uid_type type, uint8_t inst);
+
+static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
+{
+	return atomic_read(&adev->wedge_status);
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index fa301c71b782..d119de477284 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -1465,6 +1465,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 	if (!adev->accel_working)
 		return -EBUSY;
 
+	if (amdgpu_device_is_wedged(adev))
+		return -ENODEV;
+
 	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
 
 	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index ec2514079919..641b7fe24951 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -4110,6 +4110,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
 	}
 
 	adev->gpu_recovery_enabled = true;
+	atomic_set(&adev->wedge_status, 0);
 
 fence_driver_init:
 	/* Fence driver */
@@ -4362,6 +4363,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
 		amdgpu_sync_free(&adev->isolation[i].prev);
 	}
 
+	if (atomic_xchg(&adev->wedge_status, 0))
+		pm_runtime_put_autosuspend(adev->dev);
+
 	amdgpu_reset_fini(adev);
 
 	/* free i2c buses */
@@ -5155,6 +5159,10 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
 	r = 0;
 	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
 		amdgpu_set_init_level(tmp_adev, init_level);
+
+		if (atomic_xchg(&tmp_adev->wedge_status, 0))
+			pm_runtime_put_autosuspend(tmp_adev->dev);
+
 		if (full_reset) {
 			/* post card */
 			amdgpu_reset_set_dpc_status(tmp_adev, false);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index fb60a99f43c1..617d4eec4e26 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -459,6 +459,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
 	uint32_t handle, initial_domain;
 	int r;
 
+	if (amdgpu_device_is_wedged(adev))
+		return -ENODEV;
+
 	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
 
 	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
@@ -961,6 +964,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
 		abo = NULL;
 	}
 
+	if (amdgpu_device_is_wedged(adev)) {
+		r = -ENODEV;
+		goto error_put_gobj;
+	}
+
 	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
 
 	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 4204c3c750a8..53fa9fa83ae7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -730,6 +730,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	struct amdgpu_job *job;
 	int r;
 
+	if (amdgpu_device_is_wedged(adev))
+		return;
+
 	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
 
 	if (!hub->sdma_invalidation_workaround || vmid ||
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index 0b8d0a79dd45..f72c9c690c3b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -24,6 +24,7 @@
 #include <linux/kthread.h>
 #include <linux/wait.h>
 #include <linux/sched.h>
+#include <linux/pm_runtime.h>
 
 #include <drm/drm_drv.h>
 
@@ -185,9 +186,35 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 		if (r)
 			dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
 	} else {
-		drm_sched_suspend_timeout(&ring->sched);
-		if (amdgpu_sriov_vf(adev))
+		if (amdgpu_sriov_vf(adev)) {
+			drm_sched_suspend_timeout(&ring->sched);
 			adev->virt.tdr_debug = true;
+		} else {
+			struct drm_gpu_scheduler *sched;
+			struct amdgpu_fence *guilty_fence;
+
+			/* Declare the device as wedged if it's not already. */
+			if (!atomic_xchg(&adev->wedge_status, 1)) {
+				/* This ref will be dropped in amdgpu_device_reinit_after_reset. */
+				pm_runtime_get_noresume(adev->dev);
+
+				pci_clear_master(adev->pdev);
+
+				drm_dev_wedged_event(&adev->ddev, DRM_WEDGE_RECOVERY_REBIND, info);
+			}
+
+			guilty_fence = to_amdgpu_job(s_job)->hw_fence;
+
+			sched = &ring->sched;
+
+			/* Prevent anybody else from touching the ring buffer. */
+			drm_sched_wqueue_stop(sched);
+
+			amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
+
+			/* Start the scheduler again */
+			drm_sched_wqueue_start(sched);
+		}
 	}
 
 exit:
@@ -439,7 +466,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
 
 	/* Skip job if VRAM is lost and never resubmit gangs */
 	if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
-	    (job->job_run_counter && job->gang_submit))
+	    (job->job_run_counter && job->gang_submit) ||
+	    amdgpu_device_is_wedged(adev))
 		dma_fence_set_error(finished, -ECANCELED);
 
 	if (finished->error < 0) {
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index 14a72a406d37..2393cad36ac0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -781,6 +781,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
 	}
 	case AMDGPU_INFO_TIMESTAMP: {
+		if (amdgpu_device_is_wedged(adev))
+			return -ENODEV;
+
 		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
 
 		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
@@ -913,6 +916,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		if (info->read_mmr_reg.count > 128)
 			return -EINVAL;
 
+		if (amdgpu_device_is_wedged(adev))
+			return -ENODEV;
+
 		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
 
 		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
@@ -953,6 +959,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		uint64_t vm_size;
 		uint32_t pcie_gen_mask, pcie_width_mask;
 
+		if (amdgpu_device_is_wedged(adev))
+			return -ENODEV;
+
 		/* The clock limits are only available while the SMU is up. */
 		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
 
@@ -1187,6 +1196,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
 		}
 	}
 	case AMDGPU_INFO_SENSOR: {
+		if (amdgpu_device_is_wedged(adev))
+			return -ENODEV;
+
 		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
 
 		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index ac18c2be6ecb..b95dddeaccc5 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -2424,8 +2424,12 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
 		for (i = 0; i < p->n_pdds; i++) {
 			struct kfd_process_device *pdd = p->pdds[i];
 
-			if (pdd->user_gpu_id == gpu_id)
+			if (pdd->user_gpu_id == gpu_id) {
+				if (amdgpu_device_is_wedged(pdd->dev->adev))
+					return NULL;
+
 				return pdd;
+			}
 		}
 	}
 	return NULL;
-- 
2.43.0

[PATCH v3 5/5] drm/amdgpu: introduce amdgpu_dev_enter / exit

[Pierre-Eric Pelloux-Prayer]

And use them instead of drm_dev_enter / exit.

Every place that needs the GPU should already call the drm variant.
The amdgpu version also checks for the wedge status of the device and
amdgpu_device_skip_hw_access.

Direct uses of amdgpu_device_is_wedged are still ok for paths where
drm_dev_enter is used like drm ioctls.

Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
---
 drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 17 +++++++++++++++++
 drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c   | 16 ++++++++--------
 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  6 +++---
 drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    | 18 +++++++++++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_object.c |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c    | 10 +++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c    | 12 ++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c    | 12 ++++++------
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c     | 10 +++++-----
 drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c  |  4 ++--
 drivers/gpu/drm/amd/amdgpu/psp_v11_0.c     |  6 +++---
 drivers/gpu/drm/amd/amdgpu/psp_v13_0.c     |  6 +++---
 drivers/gpu/drm/amd/amdgpu/psp_v14_0.c     |  6 +++---
 drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c   |  6 +++---
 drivers/gpu/drm/amd/amdgpu/vce_v4_0.c      |  8 ++++----
 drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c      |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c      |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c      |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c      |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c    |  4 ++--
 drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c    |  4 ++--
 30 files changed, 120 insertions(+), 89 deletions(-)

diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index cd4959486c94..dd9271b20832 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -50,6 +50,7 @@
 #include <drm/ttm/ttm_placement.h>
 
 #include <drm/amdgpu_drm.h>
+#include <drm/drm_drv.h>
 #include <drm/drm_gem.h>
 #include <drm/drm_ioctl.h>
 
@@ -1387,4 +1388,20 @@ static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
 	return atomic_read(&adev->wedge_status);
 }
 
+static inline bool amdgpu_dev_enter(struct amdgpu_device *adev, int *idx)
+{
+	if (amdgpu_device_is_wedged(adev))
+		return false;
+
+	if (amdgpu_device_skip_hw_access(adev))
+		return false;
+
+	return drm_dev_enter(adev_to_drm(adev), idx);
+}
+
+static inline void amdgpu_dev_exit(struct amdgpu_device *adev, int idx)
+{
+	drm_dev_exit(idx);
+}
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
index 25b1cf7dc988..a8a98351f077 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
@@ -426,14 +426,14 @@ void amdgpu_ctx_fini(struct kref *ref)
 		}
 	}
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		mutex_lock(&adev->pm.stable_pstate_ctx_lock);
 		if (adev->pm.stable_pstate_ctx == ctx) {
 			__amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
 			adev->pm.stable_pstate_ctx = NULL;
 		}
 		mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	kfree(ctx);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 641b7fe24951..0da45d076bf6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -731,13 +731,13 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 	uint64_t last;
 	int idx;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) {
 		dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n",
 			pos, size);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 		return;
 	}
 
@@ -757,7 +757,7 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 	}
 
 	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 #ifdef CONFIG_64BIT
@@ -3025,12 +3025,18 @@ static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
 {
 	struct amdgpu_device *adev =
 		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
+	int idx;
 
 	WARN_ON_ONCE(adev->gfx.gfx_off_state);
 	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
 
+	if (!amdgpu_dev_enter(adev, &idx))
+		return;
+
 	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
 		adev->gfx.gfx_off_state = true;
+
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index 3e5634de077f..4c3aa4e4d4c9 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -3030,13 +3030,13 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
 	struct drm_device *dev = file_priv->minor->dev;
 	int idx;
 
-	if (fpriv && drm_dev_enter(dev, &idx)) {
+	if (fpriv && amdgpu_dev_enter(drm_to_adev(dev), &idx)) {
 		amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
 		amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
 		amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
 		amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
 		amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(drm_to_adev(dev), idx);
 	}
 
 	return drm_release(inode, filp);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
index c4c21dbbbdbf..62e98e50bf35 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
@@ -318,7 +318,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 	if (!adev->gart.ptr)
 		return;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
@@ -335,7 +335,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
 	}
 	amdgpu_gart_invalidate_tlb(adev);
 
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
@@ -359,7 +359,7 @@ void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
 	unsigned i, j, t;
 	int idx;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
@@ -371,7 +371,7 @@ void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
 			page_base += AMDGPU_GPU_PAGE_SIZE;
 		}
 	}
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
@@ -400,7 +400,7 @@ void amdgpu_gart_map_vram_range(struct amdgpu_device *adev, uint64_t pa,
 	/* The SYSTEM flag indicates the pages aren't in VRAM. */
 	WARN_ON_ONCE(flags & AMDGPU_PTE_SYSTEM);
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	page_base = pa;
@@ -411,7 +411,7 @@ void amdgpu_gart_map_vram_range(struct amdgpu_device *adev, uint64_t pa,
 		}
 	}
 
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
@@ -440,7 +440,7 @@ void amdgpu_gart_map_gfx9_mqd(struct amdgpu_device *adev, uint64_t offset,
 	if (!adev->gart.ptr)
 		return;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	t = offset / AMDGPU_GPU_PAGE_SIZE;
@@ -455,7 +455,7 @@ void amdgpu_gart_map_gfx9_mqd(struct amdgpu_device *adev, uint64_t offset,
 			page_base += AMDGPU_GPU_PAGE_SIZE;
 		}
 	}
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 617d4eec4e26..f29a36eae6a3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -128,12 +128,12 @@ static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
 	if (ret)
 		return ret;
 
-	if (drm_dev_enter(ddev, &idx)) {
+	if (amdgpu_dev_enter(drm_to_adev(ddev), &idx)) {
 		pm_runtime_get_noresume(ddev->dev);
 
 		ret = amdgpu_bo_fault_reserve_notify(bo);
 		if (ret) {
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(drm_to_adev(ddev), idx);
 			goto unlock;
 		}
 
@@ -141,7 +141,7 @@ static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
 					       TTM_BO_VM_NUM_PREFAULT);
 
 		pm_runtime_put_noidle(ddev->dev);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(drm_to_adev(ddev), idx);
 	} else {
 		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
 	}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
index 53fa9fa83ae7..94a77bf14c76 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
@@ -728,9 +728,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
 	struct dma_fence *fence;
 	struct amdgpu_job *job;
-	int r;
+	int r, idx;
 
-	if (amdgpu_device_is_wedged(adev))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
@@ -742,8 +742,10 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 		 * A GPU reset should flush all TLBs anyway, so no need to do
 		 * this while one is ongoing.
 		 */
-		if (!down_read_trylock(&adev->reset_domain->sem))
+		if (!down_read_trylock(&adev->reset_domain->sem)) {
+			amdgpu_dev_exit(adev, idx);
 			return;
+		}
 
 		if (adev->gmc.flush_tlb_needs_extra_type_2)
 			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
@@ -756,6 +758,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 		adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub,
 						   flush_type);
 		up_read(&adev->reset_domain->sem);
+		amdgpu_dev_exit(adev, idx);
 		return;
 	}
 
@@ -782,11 +785,13 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
 
 	dma_fence_wait(fence, false);
 	dma_fence_put(fence);
+	amdgpu_dev_exit(adev, idx);
 
 	return;
 
 error_alloc:
 	mutex_unlock(&adev->mman.default_entity.lock);
+	amdgpu_dev_exit(adev, idx);
 	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
 }
 
@@ -947,6 +952,9 @@ int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, u16 pasid,
 	unsigned long pending = 0;
 	int xcc, r = 0, err;
 
+	if (amdgpu_device_is_wedged(adev))
+		return -ENODEV;
+
 	/*
 	 * A GPU reset should flush all TLBs anyway, so no need to do
 	 * this while one is ongoing.
@@ -1283,7 +1291,7 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
 	u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo);
 	int idx;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return;
 
 	flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
@@ -1315,7 +1323,7 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
 	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0);
 	/* Requires gart_ptb_gpu_pa to be 4K aligned */
 	amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags);
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index f72c9c690c3b..d1d18c5a0f50 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -96,7 +96,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 	bool can_reset = amdgpu_gpu_recovery && adev->gpu_recovery_enabled;
 	int idx, r;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (!amdgpu_dev_enter(adev, &idx)) {
 		dev_info(adev->dev, "%s - device unplugged skipping recovery on scheduler:%s",
 			 __func__, s_job->sched->name);
 
@@ -219,7 +219,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
 
 exit:
 	amdgpu_vm_put_task_info(ti);
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	/* This is needed to add the job back to the pending list */
 	return DRM_GPU_SCHED_STAT_NO_HANG;
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 6aa895f63aea..e0c1a876d76a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -1144,12 +1144,12 @@ void amdgpu_bo_fini(struct amdgpu_device *adev)
 
 	amdgpu_ttm_fini(adev);
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
 			arch_phys_wc_del(adev->gmc.vram_mtrr);
 			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
 		}
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
index 01dca6858833..a0fbb9af818f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
@@ -5109,7 +5109,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
 		return -EBUSY;
 	}
 
-	if (!drm_dev_enter(ddev, &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return -ENODEV;
 
 	ret = amdgpu_ucode_request(adev, &usbc_pd_fw, AMDGPU_UCODE_REQUIRED,
@@ -5142,7 +5142,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
 		count = ret;
 	}
 
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	return count;
 }
 
@@ -5150,19 +5150,19 @@ int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
 {
 	int idx;
 
-	if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
+	if (!amdgpu_dev_enter(psp->adev, &idx))
 		return -ENODEV;
 
 	if (!bin_size || bin_size > PSP_1_MEG) {
 		dev_err(psp->adev->dev, "PSP firmware is invalid\n");
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(psp->adev, idx);
 		return -EINVAL;
 	}
 
 	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 	memcpy(psp->fw_pri_buf, start_addr, bin_size);
 
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(psp->adev, idx);
 	return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 9c3efc27b363..5209143e7c24 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -1624,7 +1624,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	if (!adev->mman.sdma_access_ptr)
 		return -EACCES;
 
-	if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!adev->mman.buffer_funcs_enabled || !amdgpu_dev_enter(adev, &idx))
 		return -ENODEV;
 
 	if (write)
@@ -1660,7 +1660,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
 	if (!(r || write))
 		memcpy(buf, adev->mman.sdma_access_ptr, len);
 out:
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index cd35ff7ca864..8668684e8e42 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -456,14 +456,14 @@ int amdgpu_uvd_prepare_suspend(struct amdgpu_device *adev)
 		if (!adev->uvd.inst[j].saved_bo)
 			return -ENOMEM;
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			/* re-write 0 since err_event_athub will corrupt VCPU buffer */
 			if (amdgpu_ras_intr_triggered())
 				memset(adev->uvd.inst[j].saved_bo, 0, size);
 			else
 				memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size);
 
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		}
 	}
 
@@ -495,9 +495,9 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 		ptr = adev->uvd.inst[i].cpu_addr;
 
 		if (adev->uvd.inst[i].saved_bo != NULL) {
-			if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+			if (amdgpu_dev_enter(adev, &idx)) {
 				memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size);
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 			}
 			kvfree(adev->uvd.inst[i].saved_bo);
 			adev->uvd.inst[i].saved_bo = NULL;
@@ -508,10 +508,10 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
 			hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
 			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
 				offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-				if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+				if (amdgpu_dev_enter(adev, &idx)) {
 					memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset,
 						    le32_to_cpu(hdr->ucode_size_bytes));
-					drm_dev_exit(idx);
+					amdgpu_dev_exit(adev, idx);
 				}
 				size -= le32_to_cpu(hdr->ucode_size_bytes);
 				ptr += le32_to_cpu(hdr->ucode_size_bytes);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 3952af9ae115..b583c4d41a1d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -337,11 +337,11 @@ int amdgpu_vce_resume(struct amdgpu_device *adev)
 	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
 	offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		memset_io(adev->vce.cpu_addr, 0, amdgpu_bo_size(adev->vce.vcpu_bo));
 		memcpy_toio(adev->vce.cpu_addr, adev->vce.fw->data + offset,
 			    adev->vce.fw->size - offset);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
index 34e1242d66ea..c286df7c6faf 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
@@ -332,9 +332,9 @@ static int amdgpu_vcn_save_vcpu_bo_inst(struct amdgpu_device *adev, int i)
 	if (!adev->vcn.inst[i].saved_bo)
 		return -ENOMEM;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size);
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	return 0;
@@ -383,9 +383,9 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev, int i)
 	ptr = adev->vcn.inst[i].cpu_addr;
 
 	if (adev->vcn.inst[i].saved_bo != NULL) {
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		}
 		kvfree(adev->vcn.inst[i].saved_bo);
 		adev->vcn.inst[i].saved_bo = NULL;
@@ -396,11 +396,11 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev, int i)
 		hdr = (const struct common_firmware_header *)adev->vcn.inst[i].fw->data;
 		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
 			offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
-			if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+			if (amdgpu_dev_enter(adev, &idx)) {
 				memcpy_toio(adev->vcn.inst[i].cpu_addr,
 					    adev->vcn.inst[i].fw->data + offset,
 					    le32_to_cpu(hdr->ucode_size_bytes));
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 			}
 			size -= le32_to_cpu(hdr->ucode_size_bytes);
 			ptr += le32_to_cpu(hdr->ucode_size_bytes);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index ea8688061237..01f00d55ad6f 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -1035,7 +1035,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 	if (list_empty(&vm->kernel.needs_update))
 		return 0;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return -ENODEV;
 
 	memset(&params, 0, sizeof(params));
@@ -1069,7 +1069,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
 		amdgpu_vm_bo_idle(entry);
 
 error:
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	return r;
 }
 
@@ -1170,12 +1170,12 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	struct amdgpu_res_cursor cursor;
 	int r, idx;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return -ENODEV;
 
 	tlb_cb = kmalloc_obj(*tlb_cb);
 	if (!tlb_cb) {
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 		return -ENOMEM;
 	}
 
@@ -1287,7 +1287,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 error_free:
 	kfree(tlb_cb);
 	amdgpu_vm_end_critical(&params);
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
index c73eb416c0e2..9d1fbddcdd7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
@@ -387,7 +387,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 	if (r)
 		return r;
 
-	if (!drm_dev_enter(adev_to_drm(adev), &idx))
+	if (!amdgpu_dev_enter(adev, &idx))
 		return -ENODEV;
 
 	r = vm->update_funcs->map_table(vmbo);
@@ -429,7 +429,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
 
 	r = vm->update_funcs->commit(&params, &vm->last_update);
 exit:
-	drm_dev_exit(idx);
+	amdgpu_dev_exit(adev, idx);
 	return r;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
index bd3bb3bfb668..4d51e9c534f0 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
@@ -547,20 +547,20 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops)
 			return -ENOMEM;
 		}
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
 			ret = psp_v11_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
 			if (ret) {
 				DRM_ERROR("Send long training msg failed.\n");
 				vfree(buf);
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 				return ret;
 			}
 
 			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
 			amdgpu_device_flush_hdp(adev, NULL);
 			vfree(buf);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		} else {
 			vfree(buf);
 			return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
index f9a3fbfc0634..43bb89d4fa33 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
@@ -635,20 +635,20 @@ static int psp_v13_0_memory_training(struct psp_context *psp, uint32_t ops)
 			return -ENOMEM;
 		}
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
 			ret = psp_v13_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
 			if (ret) {
 				DRM_ERROR("Send long training msg failed.\n");
 				vfree(buf);
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 				return ret;
 			}
 
 			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
 			amdgpu_device_flush_hdp(adev, NULL);
 			vfree(buf);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		} else {
 			vfree(buf);
 			return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
index 888c747d6c5c..1c8476fb387d 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
@@ -494,20 +494,20 @@ static int psp_v14_0_memory_training(struct psp_context *psp, uint32_t ops)
 			return -ENOMEM;
 		}
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
 			ret = psp_v14_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
 			if (ret) {
 				DRM_ERROR("Send long training msg failed.\n");
 				vfree(buf);
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 				return ret;
 			}
 
 			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
 			amdgpu_device_flush_hdp(adev, NULL);
 			vfree(buf);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		} else {
 			vfree(buf);
 			return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
index 029d4f4b0961..739fba7a4751 100644
--- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
@@ -483,20 +483,20 @@ static int psp_v15_0_3_memory_training(struct psp_context *psp, uint32_t ops)
 			return -ENOMEM;
 		}
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
 			ret = psp_v15_0_3_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
 			if (ret) {
 				DRM_ERROR("Send long training msg failed.\n");
 				vfree(buf);
-				drm_dev_exit(idx);
+				amdgpu_dev_exit(adev, idx);
 				return ret;
 			}
 
 			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
 			amdgpu_device_flush_hdp(adev, NULL);
 			vfree(buf);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		} else {
 			vfree(buf);
 			return -ENODEV;
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
index ee445d8abe47..e67f53f36f51 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
@@ -566,14 +566,14 @@ static int vce_v4_0_suspend(struct amdgpu_ip_block *ip_block)
 	if (adev->vce.vcpu_bo == NULL)
 		return 0;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
 			void *ptr = adev->vce.cpu_addr;
 
 			memcpy_fromio(adev->vce.saved_bo, ptr, size);
 		}
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	/*
@@ -616,12 +616,12 @@ static int vce_v4_0_resume(struct amdgpu_ip_block *ip_block)
 
 	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
 
-		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+		if (amdgpu_dev_enter(adev, &idx)) {
 			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
 			void *ptr = adev->vce.cpu_addr;
 
 			memcpy_toio(ptr, adev->vce.saved_bo, size);
-			drm_dev_exit(idx);
+			amdgpu_dev_exit(adev, idx);
 		}
 	} else {
 		r = amdgpu_vce_resume(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
index 7d2b9e4b913a..2e4b8b025b12 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
@@ -255,9 +255,9 @@ static int vcn_v2_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		fw_shared->present_flag_0 = 0;
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	amdgpu_virt_free_mm_table(adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
index 10ef5616b2f5..c10c3a672d5e 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
@@ -421,14 +421,14 @@ static int vcn_v2_5_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	struct amdgpu_fw_shared *fw_shared;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			if (adev->vcn.harvest_config & (1 << i))
 				continue;
 			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
 			fw_shared->present_flag_0 = 0;
 		}
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
index 0f4f4d66e2e4..6189c096e32f 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
@@ -326,7 +326,7 @@ static int vcn_v3_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_fw_shared *fw_shared;
 
@@ -337,7 +337,7 @@ static int vcn_v3_0_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sw_ring.is_enabled = false;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
index 4b7f7bbe18da..3f0185eac593 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
@@ -277,7 +277,7 @@ static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn4_fw_shared *fw_shared;
 
@@ -289,7 +289,7 @@ static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sq.is_enabled = 0;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
index 3f39aa83c319..ebc4729669c2 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
@@ -296,7 +296,7 @@ static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(&adev->ddev, &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn4_fw_shared *fw_shared;
 
@@ -304,7 +304,7 @@ static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->present_flag_0 = 0;
 			fw_shared->sq.is_enabled = cpu_to_le32(false);
 		}
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
index f30f80c1b07f..3797038d9f72 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
@@ -248,7 +248,7 @@ static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn4_fw_shared *fw_shared;
 
@@ -260,7 +260,7 @@ static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sq.is_enabled = 0;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
index 7b7c3bc9f046..5d6c3759fa72 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
@@ -214,7 +214,7 @@ static int vcn_v5_0_0_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn5_fw_shared *fw_shared;
 
@@ -226,7 +226,7 @@ static int vcn_v5_0_0_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sq.is_enabled = 0;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
index c9e990ab297d..270fb2ab42dc 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
@@ -259,7 +259,7 @@ static int vcn_v5_0_1_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn5_fw_shared *fw_shared;
 
@@ -268,7 +268,7 @@ static int vcn_v5_0_1_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sq.is_enabled = 0;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
index ed47ddd5b08c..3565772288b7 100644
--- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
@@ -179,7 +179,7 @@ static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
 	struct amdgpu_device *adev = ip_block->adev;
 	int i, r, idx;
 
-	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
+	if (amdgpu_dev_enter(adev, &idx)) {
 		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
 			struct amdgpu_vcn5_fw_shared *fw_shared;
 
@@ -188,7 +188,7 @@ static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
 			fw_shared->sq.is_enabled = 0;
 		}
 
-		drm_dev_exit(idx);
+		amdgpu_dev_exit(adev, idx);
 	}
 
 	if (amdgpu_sriov_vf(adev))
-- 
2.43.0

Re: [PATCH v3 1/5] drm/amdgpu: fix NULL deref in error path

[Christian König]

On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
> IF we hit the "error_free_root" error path, amdgpu_vm_pt_free_root
> is called which will clear vm->root.bo so calling amdgpu_bo_unreserve
> will crash.
> Since root_bo is a reference to the BO previously installed as
> vm->root.bo, use it.
> 
> Fixes: 8473bfdcb5b1 ("drm/amdgpu: fix error handling in amdgpu_vm_init")
> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>

IIRC somebody from the community stumbled over that already and I already reviewed the patch.

Looks like it was missed to be picked up to amd-staging-drm-next.

Anyway that one here is Reviewed-by: Christian König <christian.koenig@amd.com>

Regards,
Christian.

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index 000c982816a7..ea8688061237 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -2716,7 +2716,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  
>  error_free_root:
>  	amdgpu_vm_pt_free_root(adev, vm);
> -	amdgpu_bo_unreserve(vm->root.bo);
> +	amdgpu_bo_unreserve(root_bo);
>  	amdgpu_bo_unref(&root_bo);
>  
>  error_free_delayed:

Re: [PATCH v3 2/5] drm/amdgpu: add gpu_recovery_enabled flag to amdgpu_device

[Christian König]

On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
> Add a per-device boolean to control whether GPU recovery is attempted
> on a hang, independently of the global amdgpu.gpu_recovery module
> parameter. It defaults to true and is exposed as a write to the
> existing amdgpu_gpu_recover debugfs file, so a single device can have
> auto-recovery disabled without affecting every other GPU in the
> system.
> 
> amdgpu_device_should_recover_gpu() now takes this flag into account.
> 
> Assisted-by: LLM
> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>

Why add a new flag? Having the recovery method as "NONE" should be handled with wedge events.

I think we really need to get away from this approach that we don't recover at and leave dma_fences unsignaled for debugging.

Regards,
Christian.

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h        |  7 +++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  5 +++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c  | 21 +++++++++++++++++----
>  3 files changed, 29 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 5a32b54b1e9b..34bbb30b6f0c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -982,6 +982,13 @@ struct amdgpu_device {
>  
>  	struct amdgpu_uma_carveout_info uma_info;
>  
> +	/* Whether this device can attempt GPU recovery on a hang.
> +	 * Defaults to true; can be turned off per-device (e.g. via the
> +	 * amdgpu_gpu_recover debugfs file) independently of the global
> +	 * amdgpu.gpu_recovery module parameter.
> +	 */
> +	bool gpu_recovery_enabled;
> +
>  	/* KFD
>  	 * Must be last --ends in a flexible-array member.
>  	 */
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 5069042b5279..ec2514079919 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -4109,6 +4109,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>  		}
>  	}
>  
> +	adev->gpu_recovery_enabled = true;
> +
>  fence_driver_init:
>  	/* Fence driver */
>  	r = amdgpu_fence_driver_sw_init(adev);
> @@ -4908,6 +4910,9 @@ bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
>  	if (amdgpu_gpu_recovery == 0)
>  		goto disabled;
>  
> +	if (!adev->gpu_recovery_enabled)
> +		goto disabled;
> +
>  	/* Skip soft reset check in fatal error mode */
>  	if (!amdgpu_ras_is_poison_mode_supported(adev))
>  		return true;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> index bff2d38f3571..c54ff34572b0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
> @@ -980,9 +980,13 @@ static int amdgpu_debugfs_fence_info_show(struct seq_file *m, void *unused)
>  }
>  
>  /*
> - * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover
> + * amdgpu_debugfs_gpu_recover - manually trigger a gpu reset & recover,
> + * and control whether this device is allowed to auto-recover from a hang.
>   *
> - * Manually trigger a gpu reset at the next fence wait.
> + * Read triggers a gpu reset at the next fence wait.
> + *
> + * Write 0/1 to disable/enable auto GPU recovery for this device
> + * (equivalent to amdgpu.gpu_recovery=0, but scoped to this device only).
>   */
>  static int gpu_recover_get(void *data, u64 *val)
>  {
> @@ -1006,8 +1010,17 @@ static int gpu_recover_get(void *data, u64 *val)
>  	return 0;
>  }
>  
> +static int gpu_recover_set(void *data, u64 val)
> +{
> +	struct amdgpu_device *adev = (struct amdgpu_device *)data;
> +
> +	adev->gpu_recovery_enabled = !!val;
> +
> +	return 0;
> +}
> +
>  DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_fence_info);
> -DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, NULL,
> +DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_debugfs_gpu_recover_fops, gpu_recover_get, gpu_recover_set,
>  			 "%lld\n");
>  
>  static void amdgpu_debugfs_reset_work(struct work_struct *work)
> @@ -1042,7 +1055,7 @@ void amdgpu_debugfs_fence_init(struct amdgpu_device *adev)
>  	if (!amdgpu_sriov_vf(adev)) {
>  
>  		INIT_WORK(&adev->reset_work, amdgpu_debugfs_reset_work);
> -		debugfs_create_file("amdgpu_gpu_recover", 0444, root, adev,
> +		debugfs_create_file("amdgpu_gpu_recover", 0644, root, adev,
>  				    &amdgpu_debugfs_gpu_recover_fops);
>  	}
>  #endif

Re: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation

[Christian König]

On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
> When a job times out the kernel tries to reset the GPU to get
> the system back to a normal state.
> Sometimes it's useful to disable this process - for instance to
> be able to inspect the hardware state at the time of the hang.
> Since leaving fence unsignalled might affect the system's
> stability, this commit enables the drm wedge framework for amdgpu.
> 
> When a hang condition is detected, the GPU isn't reset but all
> the pending fences are signalled and the hung GPU cannot receive
> new work. For KFD it's implemented by making
> kfd_process_device_data_by_id return NULL if the requested GPU
> is wedged.
> 
> When the user is done, it's possible to trigger a GPU
> reset.
> 
> The auto-recovery is still enabled by default, and to use this
> feature one has to either:
> * boot with amdgpu.gpu_recovery=0: not recommended as it disables
>   recovery for all GPUs
> * write 0 to /sys/kernel/debug/dri/X/amdgpu_gpu_recover to disable
>   auto-recovery for one GPU only
> 
> Runtime power management is disabled when the device is wedged
> (because we can't submit any work to the GPU).
> 
> Assisted-by: LLM
> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>

Looks mostly reasonable to me. 

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 13 +++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  3 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  8 +++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  8 +++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    |  3 ++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 34 ++++++++++++++++++++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 12 ++++++++
>  drivers/gpu/drm/amd/amdkfd/kfd_process.c   |  6 +++-
>  8 files changed, 83 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index 34bbb30b6f0c..cd4959486c94 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -989,6 +989,13 @@ struct amdgpu_device {
>  	 */
>  	bool gpu_recovery_enabled;
>  
> +	/* Set to 1 when the device has been declared wedged following a
> +	 * hang: it no longer accepts new work and pending fences have been
> +	 * force-signalled. Reset to 0 once the device is reset back to a
> +	 * working state.
> +	 */
> +	atomic_t wedge_status;
> +
>  	/* KFD
>  	 * Must be last --ends in a flexible-array member.
>  	 */
> @@ -1374,4 +1381,10 @@ void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
>  			   uint64_t uid);
>  uint64_t amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
>  			       enum amdgpu_uid_type type, uint8_t inst);
> +
> +static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
> +{
> +	return atomic_read(&adev->wedge_status);
> +}
> +
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> index fa301c71b782..d119de477284 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
> @@ -1465,6 +1465,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>  	if (!adev->accel_working)
>  		return -EBUSY;
>  
> +	if (amdgpu_device_is_wedged(adev))
> +		return -ENODEV;
> +
>  	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>  
>  	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index ec2514079919..641b7fe24951 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -4110,6 +4110,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>  	}
>  
>  	adev->gpu_recovery_enabled = true;
> +	atomic_set(&adev->wedge_status, 0);
>  
>  fence_driver_init:
>  	/* Fence driver */
> @@ -4362,6 +4363,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
>  		amdgpu_sync_free(&adev->isolation[i].prev);
>  	}
>  
> +	if (atomic_xchg(&adev->wedge_status, 0))
> +		pm_runtime_put_autosuspend(adev->dev);
> +
>  	amdgpu_reset_fini(adev);
>  
>  	/* free i2c buses */
> @@ -5155,6 +5159,10 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
>  	r = 0;
>  	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
>  		amdgpu_set_init_level(tmp_adev, init_level);
> +
> +		if (atomic_xchg(&tmp_adev->wedge_status, 0))
> +			pm_runtime_put_autosuspend(tmp_adev->dev);
> +
>  		if (full_reset) {
>  			/* post card */
>  			amdgpu_reset_set_dpc_status(tmp_adev, false);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index fb60a99f43c1..617d4eec4e26 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -459,6 +459,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
>  	uint32_t handle, initial_domain;
>  	int r;
>  
> +	if (amdgpu_device_is_wedged(adev))
> +		return -ENODEV;
> +
>  	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>  
>  	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
> @@ -961,6 +964,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>  		abo = NULL;
>  	}
>  
> +	if (amdgpu_device_is_wedged(adev)) {
> +		r = -ENODEV;
> +		goto error_put_gobj;
> +	}
> +
>  	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>  
>  	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 4204c3c750a8..53fa9fa83ae7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -730,6 +730,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>  	struct amdgpu_job *job;
>  	int r;
>  
> +	if (amdgpu_device_is_wedged(adev))
> +		return;
> +
>  	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
>  
>  	if (!hub->sdma_invalidation_workaround || vmid ||
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> index 0b8d0a79dd45..f72c9c690c3b 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -24,6 +24,7 @@
>  #include <linux/kthread.h>
>  #include <linux/wait.h>
>  #include <linux/sched.h>
> +#include <linux/pm_runtime.h>
>  
>  #include <drm/drm_drv.h>
>  
> @@ -185,9 +186,35 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>  		if (r)
>  			dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
>  	} else {
> -		drm_sched_suspend_timeout(&ring->sched);
> -		if (amdgpu_sriov_vf(adev))
> +		if (amdgpu_sriov_vf(adev)) {
> +			drm_sched_suspend_timeout(&ring->sched);
>  			adev->virt.tdr_debug = true;

I think we should just completely drop the tdr_debug approach from the VF handling under SRIOV.

The wedge event approach should be completely sufficient to handle all of that.

Regards,
Christian.

> +		} else {
> +			struct drm_gpu_scheduler *sched;
> +			struct amdgpu_fence *guilty_fence;
> +
> +			/* Declare the device as wedged if it's not already. */
> +			if (!atomic_xchg(&adev->wedge_status, 1)) {
> +				/* This ref will be dropped in amdgpu_device_reinit_after_reset. */
> +				pm_runtime_get_noresume(adev->dev);
> +
> +				pci_clear_master(adev->pdev);
> +
> +				drm_dev_wedged_event(&adev->ddev, DRM_WEDGE_RECOVERY_REBIND, info);
> +			}
> +
> +			guilty_fence = to_amdgpu_job(s_job)->hw_fence;
> +
> +			sched = &ring->sched;
> +
> +			/* Prevent anybody else from touching the ring buffer. */
> +			drm_sched_wqueue_stop(sched);
> +
> +			amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
> +
> +			/* Start the scheduler again */
> +			drm_sched_wqueue_start(sched);
> +		}
>  	}
>  
>  exit:
> @@ -439,7 +466,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
>  
>  	/* Skip job if VRAM is lost and never resubmit gangs */
>  	if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
> -	    (job->job_run_counter && job->gang_submit))
> +	    (job->job_run_counter && job->gang_submit) ||
> +	    amdgpu_device_is_wedged(adev))
>  		dma_fence_set_error(finished, -ECANCELED);
>  
>  	if (finished->error < 0) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> index 14a72a406d37..2393cad36ac0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
> @@ -781,6 +781,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>  		return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
>  	}
>  	case AMDGPU_INFO_TIMESTAMP: {
> +		if (amdgpu_device_is_wedged(adev))
> +			return -ENODEV;
> +
>  		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>  
>  		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
> @@ -913,6 +916,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>  		if (info->read_mmr_reg.count > 128)
>  			return -EINVAL;
>  
> +		if (amdgpu_device_is_wedged(adev))
> +			return -ENODEV;
> +
>  		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>  
>  		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
> @@ -953,6 +959,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>  		uint64_t vm_size;
>  		uint32_t pcie_gen_mask, pcie_width_mask;
>  
> +		if (amdgpu_device_is_wedged(adev))
> +			return -ENODEV;
> +
>  		/* The clock limits are only available while the SMU is up. */
>  		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>  
> @@ -1187,6 +1196,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>  		}
>  	}
>  	case AMDGPU_INFO_SENSOR: {
> +		if (amdgpu_device_is_wedged(adev))
> +			return -ENODEV;
> +
>  		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>  
>  		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> index ac18c2be6ecb..b95dddeaccc5 100644
> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
> @@ -2424,8 +2424,12 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
>  		for (i = 0; i < p->n_pdds; i++) {
>  			struct kfd_process_device *pdd = p->pdds[i];
>  
> -			if (pdd->user_gpu_id == gpu_id)
> +			if (pdd->user_gpu_id == gpu_id) {
> +				if (amdgpu_device_is_wedged(pdd->dev->adev))
> +					return NULL;
> +
>  				return pdd;
> +			}
>  		}
>  	}
>  	return NULL;

Re: [PATCH v3 5/5] drm/amdgpu: introduce amdgpu_dev_enter / exit

[Christian König]

On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
> And use them instead of drm_dev_enter / exit.
> 
> Every place that needs the GPU should already call the drm variant.
> The amdgpu version also checks for the wedge status of the device and
> amdgpu_device_skip_hw_access.
> 
> Direct uses of amdgpu_device_is_wedged are still ok for paths where
> drm_dev_enter is used like drm ioctls.
> > Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>

A guard version would be really cool to have as well, I know that it isn't trivial because of the drm index which needs to be saved but I've seen that used elsewhere so it is possible.

Anyway patch is Reviewed-by: Christian König <christian.koenig@amd.com>

Regards,
Christian.

> ---
>  drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 17 +++++++++++++++++
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++++++++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c   | 16 ++++++++--------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  6 +++---
>  drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    | 18 +++++++++++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_object.c |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c    | 10 +++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c    | 12 ++++++------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c    | 12 ++++++------
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c     | 10 +++++-----
>  drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c  |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/psp_v11_0.c     |  6 +++---
>  drivers/gpu/drm/amd/amdgpu/psp_v13_0.c     |  6 +++---
>  drivers/gpu/drm/amd/amdgpu/psp_v14_0.c     |  6 +++---
>  drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c   |  6 +++---
>  drivers/gpu/drm/amd/amdgpu/vce_v4_0.c      |  8 ++++----
>  drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c      |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c      |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c      |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c      |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c    |  4 ++--
>  drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c    |  4 ++--
>  30 files changed, 120 insertions(+), 89 deletions(-)
> 
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> index cd4959486c94..dd9271b20832 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
> @@ -50,6 +50,7 @@
>  #include <drm/ttm/ttm_placement.h>
>  
>  #include <drm/amdgpu_drm.h>
> +#include <drm/drm_drv.h>
>  #include <drm/drm_gem.h>
>  #include <drm/drm_ioctl.h>
>  
> @@ -1387,4 +1388,20 @@ static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
>  	return atomic_read(&adev->wedge_status);
>  }
>  
> +static inline bool amdgpu_dev_enter(struct amdgpu_device *adev, int *idx)
> +{
> +	if (amdgpu_device_is_wedged(adev))
> +		return false;
> +
> +	if (amdgpu_device_skip_hw_access(adev))
> +		return false;
> +
> +	return drm_dev_enter(adev_to_drm(adev), idx);
> +}
> +
> +static inline void amdgpu_dev_exit(struct amdgpu_device *adev, int idx)
> +{
> +	drm_dev_exit(idx);
> +}
> +
>  #endif
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> index 25b1cf7dc988..a8a98351f077 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c
> @@ -426,14 +426,14 @@ void amdgpu_ctx_fini(struct kref *ref)
>  		}
>  	}
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		mutex_lock(&adev->pm.stable_pstate_ctx_lock);
>  		if (adev->pm.stable_pstate_ctx == ctx) {
>  			__amdgpu_ctx_set_stable_pstate(ctx, ctx->stable_pstate);
>  			adev->pm.stable_pstate_ctx = NULL;
>  		}
>  		mutex_unlock(&adev->pm.stable_pstate_ctx_lock);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	kfree(ctx);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> index 641b7fe24951..0da45d076bf6 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
> @@ -731,13 +731,13 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
>  	uint64_t last;
>  	int idx;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	if (!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4)) {
>  		dev_err(adev->dev, "unaligned pos/size (pos=0x%llx, size=0x%zx)\n",
>  			pos, size);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  		return;
>  	}
>  
> @@ -757,7 +757,7 @@ void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
>  	}
>  
>  	spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  #ifdef CONFIG_64BIT
> @@ -3025,12 +3025,18 @@ static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
>  {
>  	struct amdgpu_device *adev =
>  		container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
> +	int idx;
>  
>  	WARN_ON_ONCE(adev->gfx.gfx_off_state);
>  	WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
>  
> +	if (!amdgpu_dev_enter(adev, &idx))
> +		return;
> +
>  	if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true, 0))
>  		adev->gfx.gfx_off_state = true;
> +
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> index 3e5634de077f..4c3aa4e4d4c9 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
> @@ -3030,13 +3030,13 @@ static int amdgpu_drm_release(struct inode *inode, struct file *filp)
>  	struct drm_device *dev = file_priv->minor->dev;
>  	int idx;
>  
> -	if (fpriv && drm_dev_enter(dev, &idx)) {
> +	if (fpriv && amdgpu_dev_enter(drm_to_adev(dev), &idx)) {
>  		amdgpu_evf_mgr_shutdown(&fpriv->evf_mgr);
>  		amdgpu_userq_mgr_cancel_resume(&fpriv->userq_mgr);
>  		amdgpu_evf_mgr_flush_suspend(&fpriv->evf_mgr);
>  		amdgpu_userq_mgr_fini(&fpriv->userq_mgr);
>  		amdgpu_evf_mgr_fini(&fpriv->evf_mgr);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(drm_to_adev(dev), idx);
>  	}
>  
>  	return drm_release(inode, filp);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> index c4c21dbbbdbf..62e98e50bf35 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gart.c
> @@ -318,7 +318,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
>  	if (!adev->gart.ptr)
>  		return;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	t = offset / AMDGPU_GPU_PAGE_SIZE;
> @@ -335,7 +335,7 @@ void amdgpu_gart_unbind(struct amdgpu_device *adev, uint64_t offset,
>  	}
>  	amdgpu_gart_invalidate_tlb(adev);
>  
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> @@ -359,7 +359,7 @@ void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
>  	unsigned i, j, t;
>  	int idx;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	t = offset / AMDGPU_GPU_PAGE_SIZE;
> @@ -371,7 +371,7 @@ void amdgpu_gart_map(struct amdgpu_device *adev, uint64_t offset,
>  			page_base += AMDGPU_GPU_PAGE_SIZE;
>  		}
>  	}
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> @@ -400,7 +400,7 @@ void amdgpu_gart_map_vram_range(struct amdgpu_device *adev, uint64_t pa,
>  	/* The SYSTEM flag indicates the pages aren't in VRAM. */
>  	WARN_ON_ONCE(flags & AMDGPU_PTE_SYSTEM);
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	page_base = pa;
> @@ -411,7 +411,7 @@ void amdgpu_gart_map_vram_range(struct amdgpu_device *adev, uint64_t pa,
>  		}
>  	}
>  
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> @@ -440,7 +440,7 @@ void amdgpu_gart_map_gfx9_mqd(struct amdgpu_device *adev, uint64_t offset,
>  	if (!adev->gart.ptr)
>  		return;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	t = offset / AMDGPU_GPU_PAGE_SIZE;
> @@ -455,7 +455,7 @@ void amdgpu_gart_map_gfx9_mqd(struct amdgpu_device *adev, uint64_t offset,
>  			page_base += AMDGPU_GPU_PAGE_SIZE;
>  		}
>  	}
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> index 617d4eec4e26..f29a36eae6a3 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
> @@ -128,12 +128,12 @@ static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
>  	if (ret)
>  		return ret;
>  
> -	if (drm_dev_enter(ddev, &idx)) {
> +	if (amdgpu_dev_enter(drm_to_adev(ddev), &idx)) {
>  		pm_runtime_get_noresume(ddev->dev);
>  
>  		ret = amdgpu_bo_fault_reserve_notify(bo);
>  		if (ret) {
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(drm_to_adev(ddev), idx);
>  			goto unlock;
>  		}
>  
> @@ -141,7 +141,7 @@ static vm_fault_t amdgpu_gem_fault(struct vm_fault *vmf)
>  					       TTM_BO_VM_NUM_PREFAULT);
>  
>  		pm_runtime_put_noidle(ddev->dev);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(drm_to_adev(ddev), idx);
>  	} else {
>  		ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot);
>  	}
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> index 53fa9fa83ae7..94a77bf14c76 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
> @@ -728,9 +728,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>  	struct amdgpu_vmhub *hub = &adev->vmhub[vmhub];
>  	struct dma_fence *fence;
>  	struct amdgpu_job *job;
> -	int r;
> +	int r, idx;
>  
> -	if (amdgpu_device_is_wedged(adev))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
> @@ -742,8 +742,10 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>  		 * A GPU reset should flush all TLBs anyway, so no need to do
>  		 * this while one is ongoing.
>  		 */
> -		if (!down_read_trylock(&adev->reset_domain->sem))
> +		if (!down_read_trylock(&adev->reset_domain->sem)) {
> +			amdgpu_dev_exit(adev, idx);
>  			return;
> +		}
>  
>  		if (adev->gmc.flush_tlb_needs_extra_type_2)
>  			adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid,
> @@ -756,6 +758,7 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>  		adev->gmc.gmc_funcs->flush_gpu_tlb(adev, vmid, vmhub,
>  						   flush_type);
>  		up_read(&adev->reset_domain->sem);
> +		amdgpu_dev_exit(adev, idx);
>  		return;
>  	}
>  
> @@ -782,11 +785,13 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>  
>  	dma_fence_wait(fence, false);
>  	dma_fence_put(fence);
> +	amdgpu_dev_exit(adev, idx);
>  
>  	return;
>  
>  error_alloc:
>  	mutex_unlock(&adev->mman.default_entity.lock);
> +	amdgpu_dev_exit(adev, idx);
>  	dev_err(adev->dev, "Error flushing GPU TLB using the SDMA (%d)!\n", r);
>  }
>  
> @@ -947,6 +952,9 @@ int amdgpu_gmc_flush_gpu_tlb_pasid_xccs(struct amdgpu_device *adev, u16 pasid,
>  	unsigned long pending = 0;
>  	int xcc, r = 0, err;
>  
> +	if (amdgpu_device_is_wedged(adev))
> +		return -ENODEV;
> +
>  	/*
>  	 * A GPU reset should flush all TLBs anyway, so no need to do
>  	 * this while one is ongoing.
> @@ -1283,7 +1291,7 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
>  	u64 gart_ptb_gpu_pa = amdgpu_gmc_vram_pa(adev, adev->gart.bo);
>  	int idx;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return;
>  
>  	flags |= AMDGPU_PTE_VALID | AMDGPU_PTE_READABLE;
> @@ -1315,7 +1323,7 @@ void amdgpu_gmc_init_pdb0(struct amdgpu_device *adev)
>  	flags |= AMDGPU_PTE_SNOOPED | AMDGPU_PDE_BFS_FLAG(adev, 0);
>  	/* Requires gart_ptb_gpu_pa to be 4K aligned */
>  	amdgpu_gmc_set_pte_pde(adev, adev->gmc.ptr_pdb0, i, gart_ptb_gpu_pa, flags);
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  }
>  
>  /**
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> index f72c9c690c3b..d1d18c5a0f50 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
> @@ -96,7 +96,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>  	bool can_reset = amdgpu_gpu_recovery && adev->gpu_recovery_enabled;
>  	int idx, r;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (!amdgpu_dev_enter(adev, &idx)) {
>  		dev_info(adev->dev, "%s - device unplugged skipping recovery on scheduler:%s",
>  			 __func__, s_job->sched->name);
>  
> @@ -219,7 +219,7 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>  
>  exit:
>  	amdgpu_vm_put_task_info(ti);
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	/* This is needed to add the job back to the pending list */
>  	return DRM_GPU_SCHED_STAT_NO_HANG;
>  }
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> index 6aa895f63aea..e0c1a876d76a 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
> @@ -1144,12 +1144,12 @@ void amdgpu_bo_fini(struct amdgpu_device *adev)
>  
>  	amdgpu_ttm_fini(adev);
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
>  			arch_phys_wc_del(adev->gmc.vram_mtrr);
>  			arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
>  		}
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  }
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> index 01dca6858833..a0fbb9af818f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c
> @@ -5109,7 +5109,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
>  		return -EBUSY;
>  	}
>  
> -	if (!drm_dev_enter(ddev, &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return -ENODEV;
>  
>  	ret = amdgpu_ucode_request(adev, &usbc_pd_fw, AMDGPU_UCODE_REQUIRED,
> @@ -5142,7 +5142,7 @@ static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
>  		count = ret;
>  	}
>  
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	return count;
>  }
>  
> @@ -5150,19 +5150,19 @@ int psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
>  {
>  	int idx;
>  
> -	if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
> +	if (!amdgpu_dev_enter(psp->adev, &idx))
>  		return -ENODEV;
>  
>  	if (!bin_size || bin_size > PSP_1_MEG) {
>  		dev_err(psp->adev->dev, "PSP firmware is invalid\n");
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(psp->adev, idx);
>  		return -EINVAL;
>  	}
>  
>  	memset(psp->fw_pri_buf, 0, PSP_1_MEG);
>  	memcpy(psp->fw_pri_buf, start_addr, bin_size);
>  
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(psp->adev, idx);
>  	return 0;
>  }
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> index 9c3efc27b363..5209143e7c24 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
> @@ -1624,7 +1624,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
>  	if (!adev->mman.sdma_access_ptr)
>  		return -EACCES;
>  
> -	if (!adev->mman.buffer_funcs_enabled || !drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!adev->mman.buffer_funcs_enabled || !amdgpu_dev_enter(adev, &idx))
>  		return -ENODEV;
>  
>  	if (write)
> @@ -1660,7 +1660,7 @@ static int amdgpu_ttm_access_memory_sdma(struct ttm_buffer_object *bo,
>  	if (!(r || write))
>  		memcpy(buf, adev->mman.sdma_access_ptr, len);
>  out:
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	return r;
>  }
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> index cd35ff7ca864..8668684e8e42 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
> @@ -456,14 +456,14 @@ int amdgpu_uvd_prepare_suspend(struct amdgpu_device *adev)
>  		if (!adev->uvd.inst[j].saved_bo)
>  			return -ENOMEM;
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			/* re-write 0 since err_event_athub will corrupt VCPU buffer */
>  			if (amdgpu_ras_intr_triggered())
>  				memset(adev->uvd.inst[j].saved_bo, 0, size);
>  			else
>  				memcpy_fromio(adev->uvd.inst[j].saved_bo, ptr, size);
>  
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		}
>  	}
>  
> @@ -495,9 +495,9 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
>  		ptr = adev->uvd.inst[i].cpu_addr;
>  
>  		if (adev->uvd.inst[i].saved_bo != NULL) {
> -			if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +			if (amdgpu_dev_enter(adev, &idx)) {
>  				memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size);
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  			}
>  			kvfree(adev->uvd.inst[i].saved_bo);
>  			adev->uvd.inst[i].saved_bo = NULL;
> @@ -508,10 +508,10 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev)
>  			hdr = (const struct common_firmware_header *)adev->uvd.fw->data;
>  			if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
>  				offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
> -				if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +				if (amdgpu_dev_enter(adev, &idx)) {
>  					memcpy_toio(adev->uvd.inst[i].cpu_addr, adev->uvd.fw->data + offset,
>  						    le32_to_cpu(hdr->ucode_size_bytes));
> -					drm_dev_exit(idx);
> +					amdgpu_dev_exit(adev, idx);
>  				}
>  				size -= le32_to_cpu(hdr->ucode_size_bytes);
>  				ptr += le32_to_cpu(hdr->ucode_size_bytes);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> index 3952af9ae115..b583c4d41a1d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
> @@ -337,11 +337,11 @@ int amdgpu_vce_resume(struct amdgpu_device *adev)
>  	hdr = (const struct common_firmware_header *)adev->vce.fw->data;
>  	offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		memset_io(adev->vce.cpu_addr, 0, amdgpu_bo_size(adev->vce.vcpu_bo));
>  		memcpy_toio(adev->vce.cpu_addr, adev->vce.fw->data + offset,
>  			    adev->vce.fw->size - offset);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	return 0;
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> index 34e1242d66ea..c286df7c6faf 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c
> @@ -332,9 +332,9 @@ static int amdgpu_vcn_save_vcpu_bo_inst(struct amdgpu_device *adev, int i)
>  	if (!adev->vcn.inst[i].saved_bo)
>  		return -ENOMEM;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		memcpy_fromio(adev->vcn.inst[i].saved_bo, ptr, size);
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	return 0;
> @@ -383,9 +383,9 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev, int i)
>  	ptr = adev->vcn.inst[i].cpu_addr;
>  
>  	if (adev->vcn.inst[i].saved_bo != NULL) {
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			memcpy_toio(ptr, adev->vcn.inst[i].saved_bo, size);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		}
>  		kvfree(adev->vcn.inst[i].saved_bo);
>  		adev->vcn.inst[i].saved_bo = NULL;
> @@ -396,11 +396,11 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev, int i)
>  		hdr = (const struct common_firmware_header *)adev->vcn.inst[i].fw->data;
>  		if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) {
>  			offset = le32_to_cpu(hdr->ucode_array_offset_bytes);
> -			if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +			if (amdgpu_dev_enter(adev, &idx)) {
>  				memcpy_toio(adev->vcn.inst[i].cpu_addr,
>  					    adev->vcn.inst[i].fw->data + offset,
>  					    le32_to_cpu(hdr->ucode_size_bytes));
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  			}
>  			size -= le32_to_cpu(hdr->ucode_size_bytes);
>  			ptr += le32_to_cpu(hdr->ucode_size_bytes);
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> index ea8688061237..01f00d55ad6f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
> @@ -1035,7 +1035,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>  	if (list_empty(&vm->kernel.needs_update))
>  		return 0;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return -ENODEV;
>  
>  	memset(&params, 0, sizeof(params));
> @@ -1069,7 +1069,7 @@ int amdgpu_vm_update_pdes(struct amdgpu_device *adev,
>  		amdgpu_vm_bo_idle(entry);
>  
>  error:
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	return r;
>  }
>  
> @@ -1170,12 +1170,12 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  	struct amdgpu_res_cursor cursor;
>  	int r, idx;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return -ENODEV;
>  
>  	tlb_cb = kmalloc_obj(*tlb_cb);
>  	if (!tlb_cb) {
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  		return -ENOMEM;
>  	}
>  
> @@ -1287,7 +1287,7 @@ int amdgpu_vm_update_range(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  error_free:
>  	kfree(tlb_cb);
>  	amdgpu_vm_end_critical(&params);
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	return r;
>  }
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> index c73eb416c0e2..9d1fbddcdd7c 100644
> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm_pt.c
> @@ -387,7 +387,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  	if (r)
>  		return r;
>  
> -	if (!drm_dev_enter(adev_to_drm(adev), &idx))
> +	if (!amdgpu_dev_enter(adev, &idx))
>  		return -ENODEV;
>  
>  	r = vm->update_funcs->map_table(vmbo);
> @@ -429,7 +429,7 @@ int amdgpu_vm_pt_clear(struct amdgpu_device *adev, struct amdgpu_vm *vm,
>  
>  	r = vm->update_funcs->commit(&params, &vm->last_update);
>  exit:
> -	drm_dev_exit(idx);
> +	amdgpu_dev_exit(adev, idx);
>  	return r;
>  }
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
> index bd3bb3bfb668..4d51e9c534f0 100644
> --- a/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/psp_v11_0.c
> @@ -547,20 +547,20 @@ static int psp_v11_0_memory_training(struct psp_context *psp, uint32_t ops)
>  			return -ENOMEM;
>  		}
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
>  			ret = psp_v11_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
>  			if (ret) {
>  				DRM_ERROR("Send long training msg failed.\n");
>  				vfree(buf);
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  				return ret;
>  			}
>  
>  			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
>  			amdgpu_device_flush_hdp(adev, NULL);
>  			vfree(buf);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		} else {
>  			vfree(buf);
>  			return -ENODEV;
> diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
> index f9a3fbfc0634..43bb89d4fa33 100644
> --- a/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/psp_v13_0.c
> @@ -635,20 +635,20 @@ static int psp_v13_0_memory_training(struct psp_context *psp, uint32_t ops)
>  			return -ENOMEM;
>  		}
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
>  			ret = psp_v13_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
>  			if (ret) {
>  				DRM_ERROR("Send long training msg failed.\n");
>  				vfree(buf);
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  				return ret;
>  			}
>  
>  			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
>  			amdgpu_device_flush_hdp(adev, NULL);
>  			vfree(buf);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		} else {
>  			vfree(buf);
>  			return -ENODEV;
> diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
> index 888c747d6c5c..1c8476fb387d 100644
> --- a/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/psp_v14_0.c
> @@ -494,20 +494,20 @@ static int psp_v14_0_memory_training(struct psp_context *psp, uint32_t ops)
>  			return -ENOMEM;
>  		}
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
>  			ret = psp_v14_0_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
>  			if (ret) {
>  				DRM_ERROR("Send long training msg failed.\n");
>  				vfree(buf);
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  				return ret;
>  			}
>  
>  			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
>  			amdgpu_device_flush_hdp(adev, NULL);
>  			vfree(buf);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		} else {
>  			vfree(buf);
>  			return -ENODEV;
> diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
> index 029d4f4b0961..739fba7a4751 100644
> --- a/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
> +++ b/drivers/gpu/drm/amd/amdgpu/psp_v15_0_3.c
> @@ -483,20 +483,20 @@ static int psp_v15_0_3_memory_training(struct psp_context *psp, uint32_t ops)
>  			return -ENOMEM;
>  		}
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			memcpy_fromio(buf, adev->mman.aper_base_kaddr, sz);
>  			ret = psp_v15_0_3_memory_training_send_msg(psp, PSP_BL__DRAM_LONG_TRAIN);
>  			if (ret) {
>  				DRM_ERROR("Send long training msg failed.\n");
>  				vfree(buf);
> -				drm_dev_exit(idx);
> +				amdgpu_dev_exit(adev, idx);
>  				return ret;
>  			}
>  
>  			memcpy_toio(adev->mman.aper_base_kaddr, buf, sz);
>  			amdgpu_device_flush_hdp(adev, NULL);
>  			vfree(buf);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		} else {
>  			vfree(buf);
>  			return -ENODEV;
> diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
> index ee445d8abe47..e67f53f36f51 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c
> @@ -566,14 +566,14 @@ static int vce_v4_0_suspend(struct amdgpu_ip_block *ip_block)
>  	if (adev->vce.vcpu_bo == NULL)
>  		return 0;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
>  			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
>  			void *ptr = adev->vce.cpu_addr;
>  
>  			memcpy_fromio(adev->vce.saved_bo, ptr, size);
>  		}
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	/*
> @@ -616,12 +616,12 @@ static int vce_v4_0_resume(struct amdgpu_ip_block *ip_block)
>  
>  	if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) {
>  
> -		if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +		if (amdgpu_dev_enter(adev, &idx)) {
>  			unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo);
>  			void *ptr = adev->vce.cpu_addr;
>  
>  			memcpy_toio(ptr, adev->vce.saved_bo, size);
> -			drm_dev_exit(idx);
> +			amdgpu_dev_exit(adev, idx);
>  		}
>  	} else {
>  		r = amdgpu_vce_resume(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
> index 7d2b9e4b913a..2e4b8b025b12 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_0.c
> @@ -255,9 +255,9 @@ static int vcn_v2_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	struct amdgpu_fw_shared *fw_shared = adev->vcn.inst->fw_shared.cpu_addr;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		fw_shared->present_flag_0 = 0;
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	amdgpu_virt_free_mm_table(adev);
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
> index 10ef5616b2f5..c10c3a672d5e 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v2_5.c
> @@ -421,14 +421,14 @@ static int vcn_v2_5_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	struct amdgpu_fw_shared *fw_shared;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			if (adev->vcn.harvest_config & (1 << i))
>  				continue;
>  			fw_shared = adev->vcn.inst[i].fw_shared.cpu_addr;
>  			fw_shared->present_flag_0 = 0;
>  		}
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
> index 0f4f4d66e2e4..6189c096e32f 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v3_0.c
> @@ -326,7 +326,7 @@ static int vcn_v3_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_fw_shared *fw_shared;
>  
> @@ -337,7 +337,7 @@ static int vcn_v3_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sw_ring.is_enabled = false;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
> index 4b7f7bbe18da..3f0185eac593 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0.c
> @@ -277,7 +277,7 @@ static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn4_fw_shared *fw_shared;
>  
> @@ -289,7 +289,7 @@ static int vcn_v4_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sq.is_enabled = 0;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
> index 3f39aa83c319..ebc4729669c2 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_3.c
> @@ -296,7 +296,7 @@ static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(&adev->ddev, &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn4_fw_shared *fw_shared;
>  
> @@ -304,7 +304,7 @@ static int vcn_v4_0_3_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->present_flag_0 = 0;
>  			fw_shared->sq.is_enabled = cpu_to_le32(false);
>  		}
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
> index f30f80c1b07f..3797038d9f72 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v4_0_5.c
> @@ -248,7 +248,7 @@ static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn4_fw_shared *fw_shared;
>  
> @@ -260,7 +260,7 @@ static int vcn_v4_0_5_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sq.is_enabled = 0;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
> index 7b7c3bc9f046..5d6c3759fa72 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_0.c
> @@ -214,7 +214,7 @@ static int vcn_v5_0_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn5_fw_shared *fw_shared;
>  
> @@ -226,7 +226,7 @@ static int vcn_v5_0_0_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sq.is_enabled = 0;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
> index c9e990ab297d..270fb2ab42dc 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_1.c
> @@ -259,7 +259,7 @@ static int vcn_v5_0_1_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn5_fw_shared *fw_shared;
>  
> @@ -268,7 +268,7 @@ static int vcn_v5_0_1_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sq.is_enabled = 0;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))
> diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
> index ed47ddd5b08c..3565772288b7 100644
> --- a/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
> +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v5_0_2.c
> @@ -179,7 +179,7 @@ static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
>  	struct amdgpu_device *adev = ip_block->adev;
>  	int i, r, idx;
>  
> -	if (drm_dev_enter(adev_to_drm(adev), &idx)) {
> +	if (amdgpu_dev_enter(adev, &idx)) {
>  		for (i = 0; i < adev->vcn.num_vcn_inst; i++) {
>  			struct amdgpu_vcn5_fw_shared *fw_shared;
>  
> @@ -188,7 +188,7 @@ static int vcn_v5_0_2_sw_fini(struct amdgpu_ip_block *ip_block)
>  			fw_shared->sq.is_enabled = 0;
>  		}
>  
> -		drm_dev_exit(idx);
> +		amdgpu_dev_exit(adev, idx);
>  	}
>  
>  	if (amdgpu_sriov_vf(adev))

Re: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation

[Pierre-Eric Pelloux-Prayer]

Le 02/11/2026 à 16:44, Christian König a écrit :
> On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
>> When a job times out the kernel tries to reset the GPU to get
>> the system back to a normal state.
>> Sometimes it's useful to disable this process - for instance to
>> be able to inspect the hardware state at the time of the hang.
>> Since leaving fence unsignalled might affect the system's
>> stability, this commit enables the drm wedge framework for amdgpu.
>>
>> When a hang condition is detected, the GPU isn't reset but all
>> the pending fences are signalled and the hung GPU cannot receive
>> new work. For KFD it's implemented by making
>> kfd_process_device_data_by_id return NULL if the requested GPU
>> is wedged.
>>
>> When the user is done, it's possible to trigger a GPU
>> reset.
>>
>> The auto-recovery is still enabled by default, and to use this
>> feature one has to either:
>> * boot with amdgpu.gpu_recovery=0: not recommended as it disables
>>    recovery for all GPUs
>> * write 0 to /sys/kernel/debug/dri/X/amdgpu_gpu_recover to disable
>>    auto-recovery for one GPU only
>>
>> Runtime power management is disabled when the device is wedged
>> (because we can't submit any work to the GPU).
>>
>> Assisted-by: LLM
>> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
> 
> Looks mostly reasonable to me.
> 
>> ---
>>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 13 +++++++++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  3 ++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  8 +++++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  8 +++++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    |  3 ++
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 34 ++++++++++++++++++++--
>>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 12 ++++++++
>>   drivers/gpu/drm/amd/amdkfd/kfd_process.c   |  6 +++-
>>   8 files changed, 83 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> index 34bbb30b6f0c..cd4959486c94 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>> @@ -989,6 +989,13 @@ struct amdgpu_device {
>>   	 */
>>   	bool gpu_recovery_enabled;
>>   
>> +	/* Set to 1 when the device has been declared wedged following a
>> +	 * hang: it no longer accepts new work and pending fences have been
>> +	 * force-signalled. Reset to 0 once the device is reset back to a
>> +	 * working state.
>> +	 */
>> +	atomic_t wedge_status;
>> +
>>   	/* KFD
>>   	 * Must be last --ends in a flexible-array member.
>>   	 */
>> @@ -1374,4 +1381,10 @@ void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
>>   			   uint64_t uid);
>>   uint64_t amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
>>   			       enum amdgpu_uid_type type, uint8_t inst);
>> +
>> +static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
>> +{
>> +	return atomic_read(&adev->wedge_status);
>> +}
>> +
>>   #endif
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> index fa301c71b782..d119de477284 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>> @@ -1465,6 +1465,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>   	if (!adev->accel_working)
>>   		return -EBUSY;
>>   
>> +	if (amdgpu_device_is_wedged(adev))
>> +		return -ENODEV;
>> +
>>   	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>   
>>   	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> index ec2514079919..641b7fe24951 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>> @@ -4110,6 +4110,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>>   	}
>>   
>>   	adev->gpu_recovery_enabled = true;
>> +	atomic_set(&adev->wedge_status, 0);
>>   
>>   fence_driver_init:
>>   	/* Fence driver */
>> @@ -4362,6 +4363,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
>>   		amdgpu_sync_free(&adev->isolation[i].prev);
>>   	}
>>   
>> +	if (atomic_xchg(&adev->wedge_status, 0))
>> +		pm_runtime_put_autosuspend(adev->dev);
>> +
>>   	amdgpu_reset_fini(adev);
>>   
>>   	/* free i2c buses */
>> @@ -5155,6 +5159,10 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
>>   	r = 0;
>>   	list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
>>   		amdgpu_set_init_level(tmp_adev, init_level);
>> +
>> +		if (atomic_xchg(&tmp_adev->wedge_status, 0))
>> +			pm_runtime_put_autosuspend(tmp_adev->dev);
>> +
>>   		if (full_reset) {
>>   			/* post card */
>>   			amdgpu_reset_set_dpc_status(tmp_adev, false);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> index fb60a99f43c1..617d4eec4e26 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>> @@ -459,6 +459,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
>>   	uint32_t handle, initial_domain;
>>   	int r;
>>   
>> +	if (amdgpu_device_is_wedged(adev))
>> +		return -ENODEV;
>> +
>>   	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>   
>>   	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>> @@ -961,6 +964,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>>   		abo = NULL;
>>   	}
>>   
>> +	if (amdgpu_device_is_wedged(adev)) {
>> +		r = -ENODEV;
>> +		goto error_put_gobj;
>> +	}
>> +
>>   	PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>   
>>   	r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> index 4204c3c750a8..53fa9fa83ae7 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>> @@ -730,6 +730,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>>   	struct amdgpu_job *job;
>>   	int r;
>>   
>> +	if (amdgpu_device_is_wedged(adev))
>> +		return;
>> +
>>   	ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
>>   
>>   	if (!hub->sdma_invalidation_workaround || vmid ||
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> index 0b8d0a79dd45..f72c9c690c3b 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>> @@ -24,6 +24,7 @@
>>   #include <linux/kthread.h>
>>   #include <linux/wait.h>
>>   #include <linux/sched.h>
>> +#include <linux/pm_runtime.h>
>>   
>>   #include <drm/drm_drv.h>
>>   
>> @@ -185,9 +186,35 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>>   		if (r)
>>   			dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
>>   	} else {
>> -		drm_sched_suspend_timeout(&ring->sched);
>> -		if (amdgpu_sriov_vf(adev))
>> +		if (amdgpu_sriov_vf(adev)) {
>> +			drm_sched_suspend_timeout(&ring->sched);
>>   			adev->virt.tdr_debug = true;
> 
> I think we should just completely drop the tdr_debug approach from the VF handling under SRIOV.


Can this be done in a follow up patchset?

Pierre-Eric

> 
> The wedge event approach should be completely sufficient to handle all of that.
> 
> Regards,
> Christian.
> 
>> +		} else {
>> +			struct drm_gpu_scheduler *sched;
>> +			struct amdgpu_fence *guilty_fence;
>> +
>> +			/* Declare the device as wedged if it's not already. */
>> +			if (!atomic_xchg(&adev->wedge_status, 1)) {
>> +				/* This ref will be dropped in amdgpu_device_reinit_after_reset. */
>> +				pm_runtime_get_noresume(adev->dev);
>> +
>> +				pci_clear_master(adev->pdev);
>> +
>> +				drm_dev_wedged_event(&adev->ddev, DRM_WEDGE_RECOVERY_REBIND, info);
>> +			}
>> +
>> +			guilty_fence = to_amdgpu_job(s_job)->hw_fence;
>> +
>> +			sched = &ring->sched;
>> +
>> +			/* Prevent anybody else from touching the ring buffer. */
>> +			drm_sched_wqueue_stop(sched);
>> +
>> +			amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
>> +
>> +			/* Start the scheduler again */
>> +			drm_sched_wqueue_start(sched);
>> +		}
>>   	}
>>   
>>   exit:
>> @@ -439,7 +466,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
>>   
>>   	/* Skip job if VRAM is lost and never resubmit gangs */
>>   	if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
>> -	    (job->job_run_counter && job->gang_submit))
>> +	    (job->job_run_counter && job->gang_submit) ||
>> +	    amdgpu_device_is_wedged(adev))
>>   		dma_fence_set_error(finished, -ECANCELED);
>>   
>>   	if (finished->error < 0) {
>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> index 14a72a406d37..2393cad36ac0 100644
>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>> @@ -781,6 +781,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>   		return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
>>   	}
>>   	case AMDGPU_INFO_TIMESTAMP: {
>> +		if (amdgpu_device_is_wedged(adev))
>> +			return -ENODEV;
>> +
>>   		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>   
>>   		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>> @@ -913,6 +916,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>   		if (info->read_mmr_reg.count > 128)
>>   			return -EINVAL;
>>   
>> +		if (amdgpu_device_is_wedged(adev))
>> +			return -ENODEV;
>> +
>>   		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>   
>>   		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>> @@ -953,6 +959,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>   		uint64_t vm_size;
>>   		uint32_t pcie_gen_mask, pcie_width_mask;
>>   
>> +		if (amdgpu_device_is_wedged(adev))
>> +			return -ENODEV;
>> +
>>   		/* The clock limits are only available while the SMU is up. */
>>   		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>   
>> @@ -1187,6 +1196,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>   		}
>>   	}
>>   	case AMDGPU_INFO_SENSOR: {
>> +		if (amdgpu_device_is_wedged(adev))
>> +			return -ENODEV;
>> +
>>   		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>   
>>   		ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>> index ac18c2be6ecb..b95dddeaccc5 100644
>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>> @@ -2424,8 +2424,12 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
>>   		for (i = 0; i < p->n_pdds; i++) {
>>   			struct kfd_process_device *pdd = p->pdds[i];
>>   
>> -			if (pdd->user_gpu_id == gpu_id)
>> +			if (pdd->user_gpu_id == gpu_id) {
>> +				if (amdgpu_device_is_wedged(pdd->dev->adev))
>> +					return NULL;
>> +
>>   				return pdd;
>> +			}
>>   		}
>>   	}
>>   	return NULL;
> 

Re: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation

[Christian König]

On 11/2/26 17:44, Pierre-Eric Pelloux-Prayer wrote:
> Le 02/11/2026 à 16:44, Christian König a écrit :
>> On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
>>> When a job times out the kernel tries to reset the GPU to get
>>> the system back to a normal state.
>>> Sometimes it's useful to disable this process - for instance to
>>> be able to inspect the hardware state at the time of the hang.
>>> Since leaving fence unsignalled might affect the system's
>>> stability, this commit enables the drm wedge framework for amdgpu.
>>>
>>> When a hang condition is detected, the GPU isn't reset but all
>>> the pending fences are signalled and the hung GPU cannot receive
>>> new work. For KFD it's implemented by making
>>> kfd_process_device_data_by_id return NULL if the requested GPU
>>> is wedged.
>>>
>>> When the user is done, it's possible to trigger a GPU
>>> reset.
>>>
>>> The auto-recovery is still enabled by default, and to use this
>>> feature one has to either:
>>> * boot with amdgpu.gpu_recovery=0: not recommended as it disables
>>>    recovery for all GPUs
>>> * write 0 to /sys/kernel/debug/dri/X/amdgpu_gpu_recover to disable
>>>    auto-recovery for one GPU only
>>>
>>> Runtime power management is disabled when the device is wedged
>>> (because we can't submit any work to the GPU).
>>>
>>> Assisted-by: LLM
>>> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
>>
>> Looks mostly reasonable to me.
>>
>>> ---
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 13 +++++++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  3 ++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  8 +++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  8 +++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    |  3 ++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 34 ++++++++++++++++++++--
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 12 ++++++++
>>>   drivers/gpu/drm/amd/amdkfd/kfd_process.c   |  6 +++-
>>>   8 files changed, 83 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> index 34bbb30b6f0c..cd4959486c94 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> @@ -989,6 +989,13 @@ struct amdgpu_device {
>>>        */
>>>       bool gpu_recovery_enabled;
>>>   +    /* Set to 1 when the device has been declared wedged following a
>>> +     * hang: it no longer accepts new work and pending fences have been
>>> +     * force-signalled. Reset to 0 once the device is reset back to a
>>> +     * working state.
>>> +     */
>>> +    atomic_t wedge_status;
>>> +
>>>       /* KFD
>>>        * Must be last --ends in a flexible-array member.
>>>        */
>>> @@ -1374,4 +1381,10 @@ void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
>>>                  uint64_t uid);
>>>   uint64_t amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
>>>                      enum amdgpu_uid_type type, uint8_t inst);
>>> +
>>> +static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
>>> +{
>>> +    return atomic_read(&adev->wedge_status);
>>> +}
>>> +
>>>   #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> index fa301c71b782..d119de477284 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> @@ -1465,6 +1465,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>       if (!adev->accel_working)
>>>           return -EBUSY;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return -ENODEV;
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index ec2514079919..641b7fe24951 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -4110,6 +4110,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>>>       }
>>>         adev->gpu_recovery_enabled = true;
>>> +    atomic_set(&adev->wedge_status, 0);
>>>     fence_driver_init:
>>>       /* Fence driver */
>>> @@ -4362,6 +4363,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
>>>           amdgpu_sync_free(&adev->isolation[i].prev);
>>>       }
>>>   +    if (atomic_xchg(&adev->wedge_status, 0))
>>> +        pm_runtime_put_autosuspend(adev->dev);
>>> +
>>>       amdgpu_reset_fini(adev);
>>>         /* free i2c buses */
>>> @@ -5155,6 +5159,10 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
>>>       r = 0;
>>>       list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
>>>           amdgpu_set_init_level(tmp_adev, init_level);
>>> +
>>> +        if (atomic_xchg(&tmp_adev->wedge_status, 0))
>>> +            pm_runtime_put_autosuspend(tmp_adev->dev);
>>> +
>>>           if (full_reset) {
>>>               /* post card */
>>>               amdgpu_reset_set_dpc_status(tmp_adev, false);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> index fb60a99f43c1..617d4eec4e26 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> @@ -459,6 +459,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
>>>       uint32_t handle, initial_domain;
>>>       int r;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return -ENODEV;
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> @@ -961,6 +964,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>>>           abo = NULL;
>>>       }
>>>   +    if (amdgpu_device_is_wedged(adev)) {
>>> +        r = -ENODEV;
>>> +        goto error_put_gobj;
>>> +    }
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> index 4204c3c750a8..53fa9fa83ae7 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> @@ -730,6 +730,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>>>       struct amdgpu_job *job;
>>>       int r;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return;
>>> +
>>>       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
>>>         if (!hub->sdma_invalidation_workaround || vmid ||
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> index 0b8d0a79dd45..f72c9c690c3b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> @@ -24,6 +24,7 @@
>>>   #include <linux/kthread.h>
>>>   #include <linux/wait.h>
>>>   #include <linux/sched.h>
>>> +#include <linux/pm_runtime.h>
>>>     #include <drm/drm_drv.h>
>>>   @@ -185,9 +186,35 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>>>           if (r)
>>>               dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
>>>       } else {
>>> -        drm_sched_suspend_timeout(&ring->sched);
>>> -        if (amdgpu_sriov_vf(adev))
>>> +        if (amdgpu_sriov_vf(adev)) {
>>> +            drm_sched_suspend_timeout(&ring->sched);
>>>               adev->virt.tdr_debug = true;
>>
>> I think we should just completely drop the tdr_debug approach from the VF handling under SRIOV.
> 
> 
> Can this be done in a follow up patchset?

No, this here will basically deprecate the existing tdr_debug approach for SRIOV.

Regards,
Christian.

> 
> Pierre-Eric
> 
>>
>> The wedge event approach should be completely sufficient to handle all of that.
>>
>> Regards,
>> Christian.
>>
>>> +        } else {
>>> +            struct drm_gpu_scheduler *sched;
>>> +            struct amdgpu_fence *guilty_fence;
>>> +
>>> +            /* Declare the device as wedged if it's not already. */
>>> +            if (!atomic_xchg(&adev->wedge_status, 1)) {
>>> +                /* This ref will be dropped in amdgpu_device_reinit_after_reset. */
>>> +                pm_runtime_get_noresume(adev->dev);
>>> +
>>> +                pci_clear_master(adev->pdev);
>>> +
>>> +                drm_dev_wedged_event(&adev->ddev, DRM_WEDGE_RECOVERY_REBIND, info);
>>> +            }
>>> +
>>> +            guilty_fence = to_amdgpu_job(s_job)->hw_fence;
>>> +
>>> +            sched = &ring->sched;
>>> +
>>> +            /* Prevent anybody else from touching the ring buffer. */
>>> +            drm_sched_wqueue_stop(sched);
>>> +
>>> +            amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
>>> +
>>> +            /* Start the scheduler again */
>>> +            drm_sched_wqueue_start(sched);
>>> +        }
>>>       }
>>>     exit:
>>> @@ -439,7 +466,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
>>>         /* Skip job if VRAM is lost and never resubmit gangs */
>>>       if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
>>> -        (job->job_run_counter && job->gang_submit))
>>> +        (job->job_run_counter && job->gang_submit) ||
>>> +        amdgpu_device_is_wedged(adev))
>>>           dma_fence_set_error(finished, -ECANCELED);
>>>         if (finished->error < 0) {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> index 14a72a406d37..2393cad36ac0 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> @@ -781,6 +781,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
>>>       }
>>>       case AMDGPU_INFO_TIMESTAMP: {
>>> +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> @@ -913,6 +916,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           if (info->read_mmr_reg.count > 128)
>>>               return -EINVAL;
>>>   +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> @@ -953,6 +959,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           uint64_t vm_size;
>>>           uint32_t pcie_gen_mask, pcie_width_mask;
>>>   +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           /* The clock limits are only available while the SMU is up. */
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>   @@ -1187,6 +1196,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           }
>>>       }
>>>       case AMDGPU_INFO_SENSOR: {
>>> +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> index ac18c2be6ecb..b95dddeaccc5 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> @@ -2424,8 +2424,12 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
>>>           for (i = 0; i < p->n_pdds; i++) {
>>>               struct kfd_process_device *pdd = p->pdds[i];
>>>   -            if (pdd->user_gpu_id == gpu_id)
>>> +            if (pdd->user_gpu_id == gpu_id) {
>>> +                if (amdgpu_device_is_wedged(pdd->dev->adev))
>>> +                    return NULL;
>>> +
>>>                   return pdd;
>>> +            }
>>>           }
>>>       }
>>>       return NULL;
>>
> 

Re: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation

[kernel test robot]

Hi Pierre-Eric,

kernel test robot noticed the following build errors:

[auto build test ERROR on next-20261028]
[cannot apply to drm-misc/drm-misc-next linus/master v7.3 v7.3-rc7 v7.3-rc6 v7.3]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]

url:    https://github.com/intel-lab-lkp/linux/commits/Pierre-Eric-Pelloux-Prayer/drm-amdgpu-add-gpu_recovery_enabled-flag-to-amdgpu_device/20261029-161949
base:   next-20261028
patch link:    https://lore.kernel.org/r/20261029152000.22001-4-pierre-eric.pelloux-prayer%40amd.com
patch subject: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation
config: loongarch-allmodconfig (https://download.01.org/0day-ci/archive/20261105/202611051237.oXlTZ7GB-lkp@intel.com/config)
compiler: clang version 19.1.7 (https://github.com/llvm/llvm-project cd708029e0b2869e80abe31ddb175f7c35361f90)
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20261105/202611051237.oXlTZ7GB-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202611051237.oXlTZ7GB-lkp@intel.com/

All errors (new ones prefixed by >>):

>> drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c:956:3: error: cannot jump from this goto statement to its label
     956 |                 goto error_put_gobj;
         |                 ^
   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c:959:54: note: jump bypasses initialization of variable with __attribute__((cleanup))
     959 |         PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
         |                                                             ^
   1 error generated.


vim +956 drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c

   869	
   870	int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
   871				  struct drm_file *filp)
   872	{
   873		const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE |
   874			AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE |
   875			AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK |
   876			AMDGPU_VM_PAGE_NOALLOC;
   877		const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE |
   878			AMDGPU_VM_PAGE_PRT;
   879	
   880		struct drm_amdgpu_gem_va *args = data;
   881		struct drm_gem_object *gobj;
   882		struct amdgpu_device *adev = drm_to_adev(dev);
   883		struct amdgpu_fpriv *fpriv = filp->driver_priv;
   884		struct amdgpu_bo *abo;
   885		struct amdgpu_bo_va *bo_va;
   886		struct drm_syncobj *timeline_syncobj = NULL;
   887		struct dma_fence_chain *timeline_chain = NULL;
   888		struct drm_exec exec;
   889		uint64_t vm_size, tmp;
   890		int r = 0;
   891	
   892		/* Validate virtual address range against reserved regions. */
   893		if (args->va_address < AMDGPU_VA_RESERVED_BOTTOM) {
   894			dev_dbg(dev->dev,
   895				"va_address 0x%llx is in reserved area 0x%llx\n",
   896				args->va_address, AMDGPU_VA_RESERVED_BOTTOM);
   897			return -EINVAL;
   898		}
   899	
   900		if (args->va_address >= AMDGPU_GMC_HOLE_START &&
   901		    args->va_address < AMDGPU_GMC_HOLE_END) {
   902			dev_dbg(dev->dev,
   903				"va_address 0x%llx is in VA hole 0x%llx-0x%llx\n",
   904				args->va_address, AMDGPU_GMC_HOLE_START,
   905				AMDGPU_GMC_HOLE_END);
   906			return -EINVAL;
   907		}
   908	
   909		args->va_address &= AMDGPU_GMC_HOLE_MASK;
   910	
   911		vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE;
   912		vm_size -= AMDGPU_VA_RESERVED_TOP;
   913		if (check_add_overflow(args->va_address, args->map_size, &tmp) || tmp > vm_size) {
   914			dev_dbg(dev->dev,
   915				"va_address 0x%llx is in top reserved area 0x%llx\n",
   916				args->va_address + args->map_size, vm_size);
   917			return -EINVAL;
   918		}
   919	
   920		if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) {
   921			dev_dbg(dev->dev, "invalid flags combination 0x%08X\n",
   922				args->flags);
   923			return -EINVAL;
   924		}
   925	
   926		/* Validate operation type. */
   927		switch (args->operation) {
   928		case AMDGPU_VA_OP_MAP:
   929		case AMDGPU_VA_OP_UNMAP:
   930		case AMDGPU_VA_OP_CLEAR:
   931		case AMDGPU_VA_OP_REPLACE:
   932			break;
   933		default:
   934			dev_dbg(dev->dev, "unsupported operation %d\n",
   935				args->operation);
   936			return -EINVAL;
   937		}
   938	
   939		if (args->flags & AMDGPU_VM_DELAY_UPDATE &&
   940		    args->vm_timeline_syncobj_out)
   941			return -EINVAL;
   942	
   943		if ((args->operation != AMDGPU_VA_OP_CLEAR) &&
   944		    !(args->flags & AMDGPU_VM_PAGE_PRT)) {
   945			gobj = drm_gem_object_lookup(filp, args->handle);
   946			if (gobj == NULL)
   947				return -ENOENT;
   948			abo = gem_to_amdgpu_bo(gobj);
   949		} else {
   950			gobj = NULL;
   951			abo = NULL;
   952		}
   953	
   954		if (amdgpu_device_is_wedged(adev)) {
   955			r = -ENODEV;
 > 956			goto error_put_gobj;
   957		}
   958	
   959		PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
   960	
   961		r = PM_RUNTIME_ACQUIRE_ERR(&lock);
   962		if (r)
   963			goto error_put_gobj;
   964	
   965		/* Add input syncobj fences (if any) for synchronization. */
   966		r = amdgpu_gem_add_input_fence(filp,
   967					       args->input_fence_syncobj_handles,
   968					       args->num_syncobj_handles);
   969		if (r)
   970			goto error_put_gobj;
   971	
   972		drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT |
   973			      DRM_EXEC_IGNORE_DUPLICATES, 0);
   974		drm_exec_until_all_locked(&exec) {
   975			if (gobj) {
   976				r = drm_exec_lock_obj(&exec, gobj);
   977				drm_exec_retry_on_contention(&exec);
   978				if (unlikely(r))
   979					goto error;
   980			}
   981	
   982			r = amdgpu_vm_lock_pd(&fpriv->vm, &exec, 2);
   983			drm_exec_retry_on_contention(&exec);
   984			if (unlikely(r))
   985				goto error;
   986		}
   987	
   988		/* Resolve the BO-VA mapping for this VM/BO combination. */
   989		if (abo) {
   990			bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo);
   991			if (!bo_va) {
   992				r = -ENOENT;
   993				goto error;
   994			}
   995		} else if (args->operation != AMDGPU_VA_OP_CLEAR) {
   996			bo_va = fpriv->prt_va;
   997		} else {
   998			bo_va = NULL;
   999		}
  1000	
  1001		/*
  1002		 * Prepare the timeline syncobj node if the user requested a VM
  1003		 * timeline update. This only allocates/looks up the syncobj and
  1004		 * chain node; the actual fence is attached later.
  1005		 */
  1006		r = amdgpu_gem_update_timeline_node(filp,
  1007						    args->vm_timeline_syncobj_out,
  1008						    args->vm_timeline_point,
  1009						    &timeline_syncobj,
  1010						    &timeline_chain);
  1011		if (r)
  1012			goto error;
  1013	
  1014		switch (args->operation) {
  1015		case AMDGPU_VA_OP_MAP:
  1016			r = amdgpu_vm_bo_map(adev, bo_va, args->va_address,
  1017					     args->offset_in_bo, args->map_size,
  1018					     args->flags);
  1019			break;
  1020		case AMDGPU_VA_OP_UNMAP:
  1021			r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address);
  1022			break;
  1023	
  1024		case AMDGPU_VA_OP_CLEAR:
  1025			r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm,
  1026							args->va_address,
  1027							args->map_size);
  1028			break;
  1029		case AMDGPU_VA_OP_REPLACE:
  1030			r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address,
  1031						     args->offset_in_bo, args->map_size,
  1032						     args->flags);
  1033			break;
  1034		default:
  1035			break;
  1036		}
  1037	
  1038		/*
  1039		 * Once the VA operation is done, update the VM and obtain the fence
  1040		 * that represents the last relevant update for this mapping. This
  1041		 * fence can then be exported to the user-visible VM timeline.
  1042		 */
  1043		if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) &&
  1044		    (!adev->debug_vm || timeline_syncobj)) {
  1045			struct dma_fence *fence;
  1046	
  1047			fence = amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va,
  1048							args->operation);
  1049			if (timeline_syncobj) {
  1050				if (!args->vm_timeline_point) {
  1051					/* Replace the existing fence when no point is given. */
  1052					drm_syncobj_replace_fence(timeline_syncobj,
  1053								  fence);
  1054				} else {
  1055					/* Attach the last-update fence at a specific point. */
  1056					drm_syncobj_add_point(timeline_syncobj,
  1057							      timeline_chain,
  1058							      fence,
  1059							      args->vm_timeline_point);
  1060					timeline_chain = NULL;
  1061				}
  1062			}
  1063			dma_fence_put(fence);
  1064	
  1065		}
  1066	
  1067	error:
  1068		dma_fence_chain_free(timeline_chain);
  1069		if (timeline_syncobj)
  1070			drm_syncobj_put(timeline_syncobj);
  1071		drm_exec_fini(&exec);
  1072	error_put_gobj:
  1073		drm_gem_object_put(gobj);
  1074		return r;
  1075	}
  1076	

--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki