Re: [PATCH v3 4/5] drm/amdgpu: add wedge event implementation

["Christian König" <christian.koenig@amd.com>]

On 11/2/26 17:44, Pierre-Eric Pelloux-Prayer wrote:
> Le 02/11/2026 à 16:44, Christian König a écrit :
>> On 10/29/26 16:19, Pierre-Eric Pelloux-Prayer wrote:
>>> When a job times out the kernel tries to reset the GPU to get
>>> the system back to a normal state.
>>> Sometimes it's useful to disable this process - for instance to
>>> be able to inspect the hardware state at the time of the hang.
>>> Since leaving fence unsignalled might affect the system's
>>> stability, this commit enables the drm wedge framework for amdgpu.
>>>
>>> When a hang condition is detected, the GPU isn't reset but all
>>> the pending fences are signalled and the hung GPU cannot receive
>>> new work. For KFD it's implemented by making
>>> kfd_process_device_data_by_id return NULL if the requested GPU
>>> is wedged.
>>>
>>> When the user is done, it's possible to trigger a GPU
>>> reset.
>>>
>>> The auto-recovery is still enabled by default, and to use this
>>> feature one has to either:
>>> * boot with amdgpu.gpu_recovery=0: not recommended as it disables
>>>    recovery for all GPUs
>>> * write 0 to /sys/kernel/debug/dri/X/amdgpu_gpu_recover to disable
>>>    auto-recovery for one GPU only
>>>
>>> Runtime power management is disabled when the device is wedged
>>> (because we can't submit any work to the GPU).
>>>
>>> Assisted-by: LLM
>>> Signed-off-by: Pierre-Eric Pelloux-Prayer <pierre-eric.pelloux-prayer@amd.com>
>>
>> Looks mostly reasonable to me.
>>
>>> ---
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu.h        | 13 +++++++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c     |  3 ++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_device.c |  8 +++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c    |  8 +++++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c    |  3 ++
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_job.c    | 34 ++++++++++++++++++++--
>>>   drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c    | 12 ++++++++
>>>   drivers/gpu/drm/amd/amdkfd/kfd_process.c   |  6 +++-
>>>   8 files changed, 83 insertions(+), 4 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> index 34bbb30b6f0c..cd4959486c94 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
>>> @@ -989,6 +989,13 @@ struct amdgpu_device {
>>>        */
>>>       bool gpu_recovery_enabled;
>>>   +    /* Set to 1 when the device has been declared wedged following a
>>> +     * hang: it no longer accepts new work and pending fences have been
>>> +     * force-signalled. Reset to 0 once the device is reset back to a
>>> +     * working state.
>>> +     */
>>> +    atomic_t wedge_status;
>>> +
>>>       /* KFD
>>>        * Must be last --ends in a flexible-array member.
>>>        */
>>> @@ -1374,4 +1381,10 @@ void amdgpu_device_set_uid(struct amdgpu_uid *uid_info,
>>>                  uint64_t uid);
>>>   uint64_t amdgpu_device_get_uid(struct amdgpu_uid *uid_info,
>>>                      enum amdgpu_uid_type type, uint8_t inst);
>>> +
>>> +static inline bool amdgpu_device_is_wedged(struct amdgpu_device *adev)
>>> +{
>>> +    return atomic_read(&adev->wedge_status);
>>> +}
>>> +
>>>   #endif
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> index fa301c71b782..d119de477284 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
>>> @@ -1465,6 +1465,9 @@ int amdgpu_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>       if (!adev->accel_working)
>>>           return -EBUSY;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return -ENODEV;
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> index ec2514079919..641b7fe24951 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
>>> @@ -4110,6 +4110,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
>>>       }
>>>         adev->gpu_recovery_enabled = true;
>>> +    atomic_set(&adev->wedge_status, 0);
>>>     fence_driver_init:
>>>       /* Fence driver */
>>> @@ -4362,6 +4363,9 @@ void amdgpu_device_fini_sw(struct amdgpu_device *adev)
>>>           amdgpu_sync_free(&adev->isolation[i].prev);
>>>       }
>>>   +    if (atomic_xchg(&adev->wedge_status, 0))
>>> +        pm_runtime_put_autosuspend(adev->dev);
>>> +
>>>       amdgpu_reset_fini(adev);
>>>         /* free i2c buses */
>>> @@ -5155,6 +5159,10 @@ int amdgpu_device_reinit_after_reset(struct amdgpu_reset_context *reset_context)
>>>       r = 0;
>>>       list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
>>>           amdgpu_set_init_level(tmp_adev, init_level);
>>> +
>>> +        if (atomic_xchg(&tmp_adev->wedge_status, 0))
>>> +            pm_runtime_put_autosuspend(tmp_adev->dev);
>>> +
>>>           if (full_reset) {
>>>               /* post card */
>>>               amdgpu_reset_set_dpc_status(tmp_adev, false);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> index fb60a99f43c1..617d4eec4e26 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
>>> @@ -459,6 +459,9 @@ int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data,
>>>       uint32_t handle, initial_domain;
>>>       int r;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return -ENODEV;
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> @@ -961,6 +964,11 @@ int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data,
>>>           abo = NULL;
>>>       }
>>>   +    if (amdgpu_device_is_wedged(adev)) {
>>> +        r = -ENODEV;
>>> +        goto error_put_gobj;
>>> +    }
>>> +
>>>       PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, lock);
>>>         r = PM_RUNTIME_ACQUIRE_ERR(&lock);
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> index 4204c3c750a8..53fa9fa83ae7 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c
>>> @@ -730,6 +730,9 @@ void amdgpu_gmc_flush_gpu_tlb(struct amdgpu_device *adev, uint32_t vmid,
>>>       struct amdgpu_job *job;
>>>       int r;
>>>   +    if (amdgpu_device_is_wedged(adev))
>>> +        return;
>>> +
>>>       ring = to_amdgpu_ring(adev->mman.buffer_funcs_scheds[0]);
>>>         if (!hub->sdma_invalidation_workaround || vmid ||
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> index 0b8d0a79dd45..f72c9c690c3b 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
>>> @@ -24,6 +24,7 @@
>>>   #include <linux/kthread.h>
>>>   #include <linux/wait.h>
>>>   #include <linux/sched.h>
>>> +#include <linux/pm_runtime.h>
>>>     #include <drm/drm_drv.h>
>>>   @@ -185,9 +186,35 @@ static enum drm_gpu_sched_stat amdgpu_job_timedout(struct drm_sched_job *s_job)
>>>           if (r)
>>>               dev_err(adev->dev, "GPU Recovery Failed: %d\n", r);
>>>       } else {
>>> -        drm_sched_suspend_timeout(&ring->sched);
>>> -        if (amdgpu_sriov_vf(adev))
>>> +        if (amdgpu_sriov_vf(adev)) {
>>> +            drm_sched_suspend_timeout(&ring->sched);
>>>               adev->virt.tdr_debug = true;
>>
>> I think we should just completely drop the tdr_debug approach from the VF handling under SRIOV.
> 
> 
> Can this be done in a follow up patchset?

No, this here will basically deprecate the existing tdr_debug approach for SRIOV.

Regards,
Christian.

> 
> Pierre-Eric
> 
>>
>> The wedge event approach should be completely sufficient to handle all of that.
>>
>> Regards,
>> Christian.
>>
>>> +        } else {
>>> +            struct drm_gpu_scheduler *sched;
>>> +            struct amdgpu_fence *guilty_fence;
>>> +
>>> +            /* Declare the device as wedged if it's not already. */
>>> +            if (!atomic_xchg(&adev->wedge_status, 1)) {
>>> +                /* This ref will be dropped in amdgpu_device_reinit_after_reset. */
>>> +                pm_runtime_get_noresume(adev->dev);
>>> +
>>> +                pci_clear_master(adev->pdev);
>>> +
>>> +                drm_dev_wedged_event(&adev->ddev, DRM_WEDGE_RECOVERY_REBIND, info);
>>> +            }
>>> +
>>> +            guilty_fence = to_amdgpu_job(s_job)->hw_fence;
>>> +
>>> +            sched = &ring->sched;
>>> +
>>> +            /* Prevent anybody else from touching the ring buffer. */
>>> +            drm_sched_wqueue_stop(sched);
>>> +
>>> +            amdgpu_fence_driver_force_completion(ring, &guilty_fence->base);
>>> +
>>> +            /* Start the scheduler again */
>>> +            drm_sched_wqueue_start(sched);
>>> +        }
>>>       }
>>>     exit:
>>> @@ -439,7 +466,8 @@ static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job)
>>>         /* Skip job if VRAM is lost and never resubmit gangs */
>>>       if (job->generation != amdgpu_vm_generation(adev, job->vm) ||
>>> -        (job->job_run_counter && job->gang_submit))
>>> +        (job->job_run_counter && job->gang_submit) ||
>>> +        amdgpu_device_is_wedged(adev))
>>>           dma_fence_set_error(finished, -ECANCELED);
>>>         if (finished->error < 0) {
>>> diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> index 14a72a406d37..2393cad36ac0 100644
>>> --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
>>> @@ -781,6 +781,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
>>>       }
>>>       case AMDGPU_INFO_TIMESTAMP: {
>>> +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> @@ -913,6 +916,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           if (info->read_mmr_reg.count > 128)
>>>               return -EINVAL;
>>>   +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> @@ -953,6 +959,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           uint64_t vm_size;
>>>           uint32_t pcie_gen_mask, pcie_width_mask;
>>>   +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           /* The clock limits are only available while the SMU is up. */
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>   @@ -1187,6 +1196,9 @@ int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
>>>           }
>>>       }
>>>       case AMDGPU_INFO_SENSOR: {
>>> +        if (amdgpu_device_is_wedged(adev))
>>> +            return -ENODEV;
>>> +
>>>           PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(dev->dev, pm);
>>>             ret = PM_RUNTIME_ACQUIRE_ERR(&pm);
>>> diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> index ac18c2be6ecb..b95dddeaccc5 100644
>>> --- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
>>> @@ -2424,8 +2424,12 @@ struct kfd_process_device *kfd_process_device_data_by_id(struct kfd_process *p,
>>>           for (i = 0; i < p->n_pdds; i++) {
>>>               struct kfd_process_device *pdd = p->pdds[i];
>>>   -            if (pdd->user_gpu_id == gpu_id)
>>> +            if (pdd->user_gpu_id == gpu_id) {
>>> +                if (amdgpu_device_is_wedged(pdd->dev->adev))
>>> +                    return NULL;
>>> +
>>>                   return pdd;
>>> +            }
>>>           }
>>>       }
>>>       return NULL;
>>
>