Hi all, Today's linux-next merge of the drm tree got conflicts in: drivers/gpu/drm/xe/xe_amc.c drivers/gpu/drm/xe/xe_configfs.h drivers/gpu/drm/xe/xe_device.c drivers/gpu/drm/xe/xe_device.h drivers/gpu/drm/xe/xe_device_types.h drivers/gpu/drm/xe/xe_drm_ras.c drivers/gpu/drm/xe/xe_gsc.c drivers/gpu/drm/xe/xe_gt.c drivers/gpu/drm/xe/xe_gt_sriov_vf.c drivers/gpu/drm/xe/xe_guc_submit.c drivers/gpu/drm/xe/xe_hwmon.c drivers/gpu/drm/xe/xe_late_bind_fw.c drivers/gpu/drm/xe/xe_mert.c drivers/gpu/drm/xe/xe_pat.c drivers/gpu/drm/xe/xe_pci.c drivers/gpu/drm/xe/xe_pci_error.c drivers/gpu/drm/xe/xe_pm.c drivers/gpu/drm/xe/xe_ras.c drivers/gpu/drm/xe/xe_survivability_mode.c between commits from the drm-fixes tree and commits from the drm tree which it didn't seem worthwhile to identify :/ . I fixed it up (see below) and can carry the fix as necessary. This is now fixed as far as linux-next is concerned, but any non trivial conflicts should be mentioned to your upstream maintainer when your tree is submitted for merging. You may also want to consider cooperating with the maintainer of the conflicting tree to minimise any particularly complex conflicts. diff --git a/drivers/gpu/drm/xe/xe_amc.c b/drivers/gpu/drm/xe/xe_amc.c index d6fe59669a74b..37694e34271a1 100644 --- a/drivers/gpu/drm/xe/xe_amc.c +++ b/drivers/gpu/drm/xe/xe_amc.c @@ -20,6 +20,7 @@ #include "xe_i2c.h" #include "xe_mmio.h" #include "xe_pm.h" +#include "xe_wedge.h" /** * DOC: Add-In Management Controller (AMC) @@ -165,8 +166,8 @@ static void xe_amc_work(struct work_struct *work) case AMC_ALERT_OOB_RESET: case AMC_ALERT_CATERR: dev_warn(amc->i2c->drm_dev, "AMC Alert: %s\n", amc_alert[alert_reason]); - xe_device_declare_wedged(i2c_client_to_xe_device(client), - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(i2c_client_to_xe_device(client), + DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); break; default: dev_warn(amc->i2c->drm_dev, "unknown AMC alert: %d\n", alert_reason); diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index c41533644c928..273e2c45fea9b 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -81,6 +81,7 @@ #include "xe_vsec.h" #include "xe_wait_user_fence.h" #include "xe_wa.h" +#include "xe_wedge.h" #include #include @@ -238,7 +239,7 @@ static long xe_drm_ioctl(struct file *file, unsigned int cmd, unsigned long arg) struct xe_device *xe = to_xe_device(file_priv->minor->dev); long ret; - if (xe_device_wedged(xe)) + if (xe_wedged(xe)) return -ECANCELED; ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); @@ -256,7 +257,7 @@ static long xe_drm_compat_ioctl(struct file *file, unsigned int cmd, unsigned lo struct xe_device *xe = to_xe_device(file_priv->minor->dev); long ret; - if (xe_device_wedged(xe)) + if (xe_wedged(xe)) return -ECANCELED; ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); @@ -373,6 +374,9 @@ bool xe_device_is_admin_only(const struct xe_device *xe) } #endif +/* Number of allocated struct xe_device */ +static atomic_t xe_device_count; + static void xe_device_destroy(struct drm_device *dev, void *dummy) { struct xe_device *xe = to_xe_device(dev); @@ -392,6 +396,9 @@ static void xe_device_destroy(struct drm_device *dev, void *dummy) destroy_workqueue(xe->destroy_wq); ttm_device_fini(&xe->ttm); + + if (atomic_dec_and_test(&xe_device_count)) + wake_up_var(&xe_device_count); } /** @@ -462,6 +469,7 @@ int xe_device_init_early(struct xe_device *xe) return err; xe_bo_dev_init(&xe->bo_device); + atomic_inc(&xe_device_count); err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); if (err) return err; @@ -668,7 +676,7 @@ static void vf_update_device_info(struct xe_device *xe) xe->info.has_heci_cscfi = 0; xe->info.has_heci_gscfi = 0; xe->info.has_i2c = 0; - xe->info.has_late_bind = 0; + xe->info.late_bind_mask = 0; xe->info.skip_guc_pc = 1; xe->info.skip_pcode = 1; xe->info.has_drm_ras = false; @@ -690,16 +698,6 @@ static int xe_device_vram_alloc(struct xe_device *xe) return 0; } -static void wedged_work(struct work_struct *work); -static void xe_device_wedged_fini(struct drm_device *drm, void *arg); - -static int xe_device_wedged_init(struct xe_device *xe) -{ - INIT_WORK(&xe->wedged.work, wedged_work); - - return drmm_add_action_or_reset(&xe->drm, xe_device_wedged_fini, xe); -} - /** * xe_device_probe_early: Device early probe * @xe: xe device instance @@ -759,16 +757,11 @@ int xe_device_probe_early(struct xe_device *xe) */ assert_lmem_ready(xe); - xe->wedged.mode = xe_device_validate_wedged_mode(xe, xe_modparam.wedged_mode) ? - XE_DEFAULT_WEDGED_MODE : xe_modparam.wedged_mode; - drm_dbg(&xe->drm, "wedged_mode: setting mode (%u) %s\n", - xe->wedged.mode, xe_wedged_mode_to_string(xe->wedged.mode)); - err = xe_device_vram_alloc(xe); if (err) return err; - err = xe_device_wedged_init(xe); + err = xe_wedge_init(xe); if (err) return err; @@ -776,6 +769,25 @@ int xe_device_probe_early(struct xe_device *xe) } ALLOW_ERROR_INJECTION(xe_device_probe_early, ERRNO); /* See xe_pci_probe() */ +static void override_has_cached_pt(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) { + /* + * Wa_16029380221: The affected GT will always use + * non-coherent access to page tables, so we must do + * uncached writes from the CPU. + */ + if (XE_GT_WA(gt, 16029380221)) + xe->info.has_cached_pt = false; + + if (XE_GT_WA(gt, 14026539277)) + xe->info.has_cached_pt = false; + } +} + static int probe_has_flat_ccs(struct xe_device *xe) { struct xe_gt *gt; @@ -858,14 +870,6 @@ static void detect_preproduction_hw(struct xe_device *xe) } } -static void xe_device_wedged_fini(struct drm_device *drm, void *arg) -{ - struct xe_device *xe = arg; - - if (atomic_read(&xe->wedged.flag)) - xe_pm_runtime_put(xe); -} - #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE static int xe_debug_page_size_alloc_ctrl_init(struct xe_device *xe) { @@ -926,14 +930,7 @@ int xe_device_probe(struct xe_device *xe) return err; } - /* - * Wa_16029380221: The affected GT will always use non-coherent - * access to page tables, so we must do uncached writes from the - * CPU. - */ - for_each_gt(gt, xe, id) - if (XE_GT_WA(gt, 16029380221)) - xe->info.has_cached_pt = false; + override_has_cached_pt(xe); for_each_tile(tile, xe, id) { err = xe_ggtt_init_early(tile->mem.ggtt); @@ -983,7 +980,9 @@ int xe_device_probe(struct xe_device *xe) if (err) return err; - xe_ras_init(xe); + err = xe_ras_init(xe); + if (err) + return err; /* * Now that GT is initialized (TTM in particular), @@ -1104,13 +1103,6 @@ int xe_device_probe(struct xe_device *xe) detect_preproduction_hw(xe); - /* - * Process and log any errors detected by hardware. Possible results can - * include declaring the device as wedged, which must be done only after - * xe_device_wedged_fini() is registered. - */ - xe_ras_process_errors(xe); - err = devm_add_action_or_reset(xe->drm.dev, xe_device_sanitize, xe); if (err) goto err_unregister_display; @@ -1346,153 +1338,6 @@ u64 xe_device_uncanonicalize_addr(struct xe_device *xe, u64 address) return address & GENMASK_ULL(xe->info.va_bits - 1, 0); } -/** - * DOC: Xe Device Wedging - * - * Xe driver uses drm device wedged uevent as documented in Documentation/gpu/drm-uapi.rst. - * When device is in wedged state, every IOCTL will be blocked and GT cannot - * be used. The conditions under which the driver declares the device wedged - * depend on the wedged mode configuration (see &enum xe_wedged_mode). The - * default recovery method for a wedged state is rebind/bus-reset. - * - * Another recovery method is vendor-specific. Below are the cases that send - * ``WEDGED=vendor-specific`` recovery method in drm device wedged uevent. - * - * Case: Firmware Flash - * -------------------- - * - * Identification Hint - * +++++++++++++++++++ - * - * ``WEDGED=vendor-specific`` drm device wedged uevent with - * :ref:`Runtime Survivability mode ` is used to notify - * admin/userspace consumer about the need for a firmware flash. - * - * Recovery Procedure - * ++++++++++++++++++ - * - * Once ``WEDGED=vendor-specific`` drm device wedged uevent is received, follow - * the below steps - * - * - Check Runtime Survivability mode sysfs. - * If enabled, firmware flash is required to recover the device. - * - * /sys/bus/pci/devices//survivability_mode - * - * - Admin/userspace consumer can use firmware flashing tools like fwupd to flash - * firmware and restore device to normal operation. - */ - -static void wedged_work(struct work_struct *work) -{ - struct xe_device *xe = container_of(work, struct xe_device, wedged.work); - - /* Notify userspace of wedged device */ - drm_dev_wedged_event(&xe->drm, xe->wedged.method, NULL); -} - -#define WEDGED_URL "https://docs.kernel.org/gpu/drm-uapi.html#device-wedging" -#define XE_BUG_URL "https://gitlab.freedesktop.org/drm/xe/kernel/issues/new" - -/** - * xe_device_declare_wedged - Declare device wedged - * @xe: xe device instance - * @method: method(s) to be used for recovery - * - * This is a final state that can only be cleared with the recovery method - * specified in the drm wedged uevent. - * - * In this state every IOCTL will be blocked so the GT cannot be used. - * In general it will be called upon any critical error such as gt reset - * failure or guc loading failure. Userspace will be notified of this state - * through device wedged uevent. - * If xe.wedged module parameter is set to 2, this function will be called - * on every single execution timeout (a.k.a. GPU hang) right after devcoredump - * snapshot capture. In this mode, GT reset won't be attempted so the state of - * the issue is preserved for further debugging. - */ -void xe_device_declare_wedged(struct xe_device *xe, unsigned long method) -{ - struct xe_gt *gt; - u8 id; - - if (xe->wedged.mode == XE_WEDGED_MODE_NEVER) { - drm_dbg(&xe->drm, "Wedged mode is forcibly disabled\n"); - return; - } - - if (!atomic_xchg(&xe->wedged.flag, 1)) { - xe->needs_flr_on_fini = true; - xe_pm_runtime_get_noresume(xe); - - xe_log_err_fatal(xe, WEDGED, -EIO, "Device declared wedged!\n"); - xe_err_once(xe, "IOCTLs and executions are now blocked!\n" - "For recovery procedure, refer to %s\n" - "Please file a _new_ bug report at %s\n", - WEDGED_URL, XE_BUG_URL); - - for_each_initialized_gt(gt, xe, id) - xe_gt_declare_wedged(gt); - - /* - * XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET is intended for debugging - * hangs, so wedge the device with 'none' recovery method and have - * it available to the user for debugging. - */ - if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) - xe->wedged.method = DRM_WEDGE_RECOVERY_NONE; - else - xe->wedged.method = method; - - schedule_work(&xe->wedged.work); - } -} - -/** - * xe_device_validate_wedged_mode - Check if given mode is supported - * @xe: the &xe_device - * @mode: requested mode to validate - * - * Check whether the provided wedged mode is supported. - * - * Return: 0 if mode is supported, error code otherwise. - */ -int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode) -{ - if (mode > XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { - drm_dbg(&xe->drm, "wedged_mode: invalid value (%u)\n", mode); - return -EINVAL; - } else if (mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET && (IS_SRIOV_VF(xe) || - (IS_SRIOV_PF(xe) && !IS_ENABLED(CONFIG_DRM_XE_DEBUG)))) { - drm_dbg(&xe->drm, "wedged_mode: (%u) %s mode is not supported for %s\n", - mode, xe_wedged_mode_to_string(mode), - xe_sriov_mode_to_string(xe_device_sriov_mode(xe))); - return -EPERM; - } - - return 0; -} - -/** - * xe_wedged_mode_to_string - Convert enum value to string. - * @mode: the &xe_wedged_mode to convert - * - * Returns: wedged mode as a user friendly string. - */ -const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode) -{ - switch (mode) { - case XE_WEDGED_MODE_NEVER: - return "never"; - case XE_WEDGED_MODE_UPON_CRITICAL_ERROR: - return "upon-critical-error"; - case XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: - return "upon-any-hang-no-reset"; - default: - return ""; - } -} - /** * xe_device_asid_to_vm() - Find VM from ASID * @xe: the &xe_device @@ -1517,3 +1362,25 @@ struct xe_vm *xe_device_asid_to_vm(struct xe_device *xe, u32 asid) return vm; } + +/** + * xe_device_exit() - Device subsystem exit function. + * + * Exit function to be called at module unload time. + */ +void xe_device_exit(void) +{ + /* + * Wait for all devices to be freed. 20s is well above the typical + * maximum dma_fence signalling time, so warn and keep waiting if + * we're still not done by then, since it may indicate a leaked + * xe_device reference is stalling module unload. + */ + if (!wait_var_event_timeout(&xe_device_count, + !atomic_read(&xe_device_count), + HZ * 20)) { + pr_warn("%s: Waiting for %d xe device(s) to be freed before unloading.\n", + DRIVER_NAME, atomic_read(&xe_device_count)); + wait_var_event(&xe_device_count, !atomic_read(&xe_device_count)); + } +} diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index 3ce1481d0e573..6df60edcb8f81 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -211,11 +211,6 @@ bool xe_device_is_l2_flush_optimized(struct xe_device *xe); void xe_device_td_flush(struct xe_device *xe); void xe_device_l2_flush(struct xe_device *xe, bool force); -static inline bool xe_device_wedged(struct xe_device *xe) -{ - return atomic_read(&xe->wedged.flag); -} - #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE static inline bool xe_debug_page_size_supported(struct xe_device *xe) { @@ -264,10 +259,6 @@ static inline bool xe_debug_page_size_mode_is_mixed(struct xe_device *xe) } #endif -void xe_device_declare_wedged(struct xe_device *xe, unsigned long method); -int xe_device_validate_wedged_mode(struct xe_device *xe, unsigned int mode); -const char *xe_wedged_mode_to_string(enum xe_wedged_mode mode); - struct xe_file *xe_file_get(struct xe_file *xef); void xe_file_put(struct xe_file *xef); @@ -286,6 +277,8 @@ static inline bool xe_device_is_admin_only(const struct xe_device *xe) } #endif +void xe_device_exit(void); + /* * Occasionally it is seen that the G2H worker starts running after a delay of more than * a second even after being queued and activated by the Linux workqueue subsystem. This diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 5aa52975ebb46..d99881c6e1d53 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -12,6 +12,7 @@ #include #include +#include "xe_cper_types.h" #include "xe_devcoredump_types.h" #include "xe_drm_ras_types.h" #include "xe_heci_gsc.h" @@ -21,6 +22,7 @@ #include "xe_platform_types.h" #include "xe_pmu_types.h" #include "xe_pt_types.h" +#include "xe_ras_types.h" #include "xe_sriov_pf_types.h" #include "xe_sriov_types.h" #include "xe_sriov_vf_types.h" @@ -30,6 +32,7 @@ #include "xe_sysctrl_types.h" #include "xe_tile_types.h" #include "xe_validation.h" +#include "xe_wedge_types.h" #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) #define TEST_VM_OPS_ERROR @@ -47,22 +50,6 @@ struct xe_pxp; struct xe_ttm_stolen_mgr; struct xe_vram_region; -/** - * enum xe_wedged_mode - possible wedged modes - * @XE_WEDGED_MODE_NEVER: Device will never be declared wedged. - * @XE_WEDGED_MODE_UPON_CRITICAL_ERROR: Device will be declared wedged only - * when critical error occurs like GT reset failure or firmware failure. - * This is the default mode. - * @XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET: Device will be declared wedged on - * any hang. In this mode, engine resets are disabled to avoid automatic - * recovery attempts. This mode is primarily intended for debugging hangs. - */ -enum xe_wedged_mode { - XE_WEDGED_MODE_NEVER = 0, - XE_WEDGED_MODE_UPON_CRITICAL_ERROR = 1, - XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET = 2, -}; - #ifdef CONFIG_DRM_XE_DEBUG_PAGE_SIZE /** * enum xe_page_size_alloc_ctrl_mode - User BO page-size allocation control modes @@ -164,6 +151,8 @@ struct xe_device { u8 vm_max_level; /** @info.va_bits: Maximum bits of a virtual address */ u8 va_bits; + /** @info.late_bind_mask: Indicates supported late binding firmwares */ + u8 late_bind_mask; /* * Keep all flags below alphabetically sorted @@ -185,6 +174,8 @@ struct xe_device { u8 has_drm_ras:1; /** @info.has_fan_control: Device supports fan control */ u8 has_fan_control:1; + /** @info.has_fixed_vram_channels: Device has fixed VRAM temperature channels */ + u8 has_fixed_vram_channels:1; /** @info.has_flat_ccs: Whether flat CCS metadata is used */ u8 has_flat_ccs:1; /** @info.has_gsc_nvm: Device has gsc non-volatile memory */ @@ -195,8 +186,6 @@ struct xe_device { u8 has_heci_gscfi:1; /** @info.has_i2c: Device has I2C controller */ u8 has_i2c:1; - /** @info.has_late_bind: Device has firmware late binding support */ - u8 has_late_bind:1; /** @info.has_llc: Device has a shared CPU+GPU last level cache */ u8 has_llc:1; /** @info.has_mbx_power_limits: Device has support to manage power limits using @@ -553,18 +542,7 @@ struct xe_device { atomic_t in_reset; /** @wedged: Struct to control Wedged States and mode */ - struct { - /** @wedged.flag: Xe device faced a critical error and is now blocked. */ - atomic_t flag; - /** @wedged.mode: Mode controlled by kernel parameter and debugfs */ - enum xe_wedged_mode mode; - /** @wedged.method: Recovery method to be sent in the drm device wedged uevent */ - unsigned long method; - /** @wedged.inconsistent_reset: Inconsistent reset policy state between GTs */ - bool inconsistent_reset; - /** @wedged.work: Worker to defer non-atomic calls */ - struct work_struct work; - } wedged; + struct xe_wedge wedged; /** @devres_group: devres group */ void *devres_group; @@ -580,8 +558,17 @@ struct xe_device { /** @pmu: performance monitoring unit */ struct xe_pmu pmu; - /** @ras: RAS structure for device */ - struct xe_drm_ras ras; + /** @ras: RAS (Reliability, Availability, Serviceability) structures */ + struct { + /** @ras.nl_data: drm-ras netlink data */ + struct xe_drm_ras nl_data; + + /** @ras.state: RAS device and firmware state */ + struct xe_ras_state state; + } ras; + + /** @cper: CPER state */ + struct xe_cper cper; /** @i2c: I2C host controller */ struct xe_i2c *i2c; diff --git a/drivers/gpu/drm/xe/xe_drm_ras.c b/drivers/gpu/drm/xe/xe_drm_ras.c index 30cf40ee5fe4d..7554803aae11a 100644 --- a/drivers/gpu/drm/xe/xe_drm_ras.c +++ b/drivers/gpu/drm/xe/xe_drm_ras.c @@ -31,7 +31,7 @@ static int query_error_counter(struct xe_device *xe, enum drm_xe_ras_error_severity severity, u32 error_id, const char **name, u32 *val) { - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct xe_drm_ras_counter *info = ras->info[severity]; if (!info || !info[error_id].name) @@ -52,7 +52,7 @@ static int clear_error_counter(struct xe_device *xe, enum drm_xe_ras_error_severity severity, u32 error_id) { - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct xe_drm_ras_counter *info = ras->info[severity]; if (!info || !info[error_id].name) @@ -101,7 +101,7 @@ static int query_correctable_error_threshold(struct drm_ras_node *ep, u32 error_ const char **name, u32 *threshold) { struct xe_device *xe = ep->priv; - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE]; if (!info || !info[error_id].name) @@ -117,7 +117,7 @@ static int query_correctable_error_threshold(struct drm_ras_node *ep, u32 error_ static int set_correctable_error_threshold(struct drm_ras_node *ep, u32 error_id, u32 threshold) { struct xe_device *xe = ep->priv; - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct xe_drm_ras_counter *info = ras->info[DRM_XE_RAS_ERR_SEV_CORRECTABLE]; if (!info || !info[error_id].name) @@ -153,7 +153,7 @@ static int assign_node_params(struct xe_device *xe, struct drm_ras_node *node, const enum drm_xe_ras_error_severity severity) { struct pci_dev *pdev = to_pci_dev(xe->drm.dev); - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; const char *device_name; device_name = kasprintf(GFP_KERNEL, "%04x:%02x:%02x.%d", @@ -201,7 +201,7 @@ static void cleanup_node(struct drm_device *drm, void *node) static int register_nodes(struct xe_device *xe) { - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct drm_ras_node *node; int i, ret; @@ -241,7 +241,7 @@ static int register_nodes(struct xe_device *xe) */ void xe_drm_ras_event(struct xe_device *xe, u32 component, u32 severity, u32 value) { - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct xe_drm_ras_counter *info; struct drm_ras_node *node; int ret; @@ -275,7 +275,7 @@ void xe_drm_ras_event(struct xe_device *xe, u32 component, u32 severity, u32 val */ int xe_drm_ras_init(struct xe_device *xe) { - struct xe_drm_ras *ras = &xe->ras; + struct xe_drm_ras *ras = &xe->ras.nl_data; struct drm_ras_node *node; int err; diff --git a/drivers/gpu/drm/xe/xe_gsc.c b/drivers/gpu/drm/xe/xe_gsc.c index 2bbe0f20ff8d3..db669a953886a 100644 --- a/drivers/gpu/drm/xe/xe_gsc.c +++ b/drivers/gpu/drm/xe/xe_gsc.c @@ -31,6 +31,7 @@ #include "xe_sched_job.h" #include "xe_uc_fw.h" #include "xe_wa.h" +#include "xe_wedge.h" #include "instructions/xe_gsc_commands.h" #include "regs/xe_gsc_regs.h" #include "regs/xe_gt_regs.h" @@ -340,8 +341,8 @@ static int gsc_er_complete(struct xe_gt *gt) * declare the device as wedged. */ xe_gt_err(gt, "GSC ER timed out!\n"); - xe_device_declare_wedged(gt_to_xe(gt), - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(gt_to_xe(gt), + DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); return -EIO; } diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 286aee5775c0d..b751b5044f3ca 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -12,7 +12,6 @@ #include #include -#include #include #include "instructions/xe_alu_commands.h" @@ -68,6 +67,7 @@ #include "xe_uc_fw.h" #include "xe_vm.h" #include "xe_wa.h" +#include "xe_wedge.h" #include "xe_wopcm.h" #define GRDOM_RESET_TIMEOUT_MS 5 @@ -459,35 +459,6 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) return err; } -static void wa_14026539277(struct xe_gt *gt) -{ - struct xe_device *xe = gt_to_xe(gt); - u32 val; - - /* - * FIXME: We currently can't use FUNC(xe_rtp_match_not_sriov_vf) in the - * rules for Wa_14026539277 due to xe_wa_process_device_oob() being - * called before xe_sriov_probe_early(); and we can't move the call to - * the former to happen after the latter because MMIO read functions - * already depend on a device OOB workaround. This needs to be fixed by - * allowing workaround checks to happen at different stages of driver - * initialization. - */ - if (IS_SRIOV_VF(xe)) - return; - - if (!XE_DEVICE_WA(xe, 14026539277)) - return; - - if (!xe_gt_is_main_type(gt)) - return; - - val = xe_gt_mcr_unicast_read_any(gt, L2COMPUTESIDECTRL); - val &= ~CECTRL; - val |= CECTRL_CENODATA_ALWAYS; - xe_gt_mcr_multicast_write(gt, L2COMPUTESIDECTRL, val); -} - int xe_gt_init_early(struct xe_gt *gt) { int err; @@ -621,15 +592,6 @@ static int gt_init_with_gt_forcewake(struct xe_gt *gt) else gt->info.has_wmtp_disabled = 0; - /* - * Wa_14026539277 can't be implemented as a regular GT workaround (i.e. - * as an entry in gt_was[]) for two reasons: it is actually a device - * workaround that happens to involve programming a GT register; and it - * needs to be applied early to avoid getting the hardware in a bad - * state before we have a chance to do the necessary programming. - */ - wa_14026539277(gt); - return 0; } @@ -948,7 +910,7 @@ static void gt_reset_worker(struct work_struct *w) unsigned int fw_ref; int err; - if (xe_device_wedged(gt_to_xe(gt))) + if (xe_wedged(gt_to_xe(gt))) goto err_pm_put; if (xe_device_is_in_reset(gt_to_xe(gt))) @@ -1007,8 +969,7 @@ static void gt_reset_worker(struct work_struct *w) err_fail: xe_log_err_fatal(gt, GT, err, "reset failed\n"); - xe_device_declare_wedged(gt_to_xe(gt), - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(gt_to_xe(gt), DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); err_pm_put: xe_pm_runtime_put(gt_to_xe(gt)); } @@ -1091,6 +1052,19 @@ int xe_gt_sanitize_freq(struct xe_gt *gt) return ret; } +/** + * xe_gt_resume_early() - Early init for GT resume + * @gt: the GT object + * + * Handles GT related init before GT restart after a suspend cycle. + */ +void xe_gt_resume_early(struct xe_gt *gt) +{ + xe_gt_idle_disable_c6(gt); + + xe_uc_resume_early(>->uc); +} + int xe_gt_resume(struct xe_gt *gt) { int err; @@ -1215,19 +1189,19 @@ static void gt_wedge_worker(struct work_struct *w) { struct xe_gt *gt = container_of(w, typeof(*gt), wedge.worker); - xe_uc_declare_wedged(>->uc); + xe_uc_wedge_declare(>->uc); xe_tlb_inval_reset(>->tlb_inval); xe_pm_runtime_put(gt_to_xe(gt)); } /** - * xe_gt_declare_wedged() - Declare GT wedged + * xe_gt_wedge_declare() - Declare GT wedged * @gt: the GT object * * Wedge the GT which stops all submission, saves desired debug state, and * cleans up anything which could timeout. */ -void xe_gt_declare_wedged(struct xe_gt *gt) +void xe_gt_wedge_declare(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c index 28ec83bb8100d..2a5f2064ad3e8 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_vf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_vf.c @@ -37,6 +37,7 @@ #include "xe_tile_sriov_vf.h" #include "xe_tlb_inval.h" #include "xe_uc_fw.h" +#include "xe_wedge.h" #include "xe_wopcm.h" #define make_u64_from_u32(hi, lo) ((u64)((u64)(u32)(hi) << 32 | (u32)(lo))) @@ -1462,7 +1463,7 @@ static void vf_post_migration_recovery(struct xe_gt *gt) fail: vf_post_migration_abort(gt); xe_gt_sriov_err(gt, "migration recovery failed (%pe)\n", ERR_PTR(err)); - xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); return; queue: diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 01f79f8883ba6..b14b37b767f5e 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -50,6 +50,7 @@ #include "xe_trace.h" #include "xe_uc_fw.h" #include "xe_vm.h" +#include "xe_wedge.h" #define XE_GUC_EXEC_QUEUE_CGP_CONTEXT_ERROR_LEN 6 @@ -286,20 +287,26 @@ static void primelockdep(struct xe_guc *guc) fs_reclaim_release(GFP_KERNEL); } +static int guc_id_mgr_init(struct xe_guc *guc) +{ + if (IS_SRIOV_PF(guc_to_xe(guc))) + return xe_guc_id_mgr_init_shared(&guc->submission_state.idm); + else if (IS_SRIOV_VF(guc_to_xe(guc))) + return xe_guc_id_mgr_init_small(&guc->submission_state.idm, + xe_gt_sriov_vf_guc_ids(guc_to_gt(guc))); + else + return xe_guc_id_mgr_init(&guc->submission_state.idm); +} + /** * xe_guc_submit_init() - Initialize GuC submission. * @guc: the &xe_guc to initialize - * @num_ids: number of GuC context IDs to use * - * The bare-metal or PF driver can pass ~0 as &num_ids to indicate that all - * GuC context IDs supported by the GuC firmware should be used for submission. - * - * Only VF drivers will have to provide explicit number of GuC context IDs - * that they can use for submission. + * This function initializes the GuC submission state. * * Return: 0 on success or a negative error code on failure. */ -int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) +int xe_guc_submit_init(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); @@ -309,7 +316,7 @@ int xe_guc_submit_init(struct xe_guc *guc, unsigned int num_ids) if (err) return err; - err = xe_guc_id_mgr_init(&guc->submission_state.idm, num_ids); + err = guc_id_mgr_init(guc); if (err) return err; @@ -415,8 +422,7 @@ static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) int ret, i; mutex_lock(&guc->submission_state.lock); - ret = xe_guc_id_mgr_reserve_locked(&guc->submission_state.idm, - q->width); + ret = xe_guc_id_mgr_reserve_usable_locked(&guc->submission_state.idm, q->width); mutex_unlock(&guc->submission_state.lock); if (ret < 0) return ret; @@ -1450,10 +1456,10 @@ static bool guc_submit_hint_wedged(struct xe_guc *guc) if (xe->wedged.mode != XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) return false; - if (xe_device_wedged(xe)) + if (xe_wedged(xe)) return true; - xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); return true; } @@ -1666,7 +1672,7 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) if (!exec_queue_killed(q)) wedged = guc_submit_hint_wedged(exec_queue_to_guc(q)); else - wedged = xe_device_wedged(xe); + wedged = xe_wedged(xe); /* * Only tag as GPU hang if this is the original timeout, not a @@ -1786,8 +1792,8 @@ guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) } if (wedge_device) - xe_device_declare_wedged(gt_to_xe(q->gt), - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(gt_to_xe(q->gt), + DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); /* * We want the job added back to the pending list so it gets freed; this @@ -2552,7 +2558,7 @@ static int guc_exec_queue_wait_suspend_done(struct xe_exec_queue *q, bool blocki WAIT_COND, HZ * 5); } - if (!blocking && vf_recovery(guc) && !xe_device_wedged(xe)) + if (!blocking && vf_recovery(guc) && !xe_wedged(xe)) return -EAGAIN; if (!ret) @@ -2812,7 +2818,7 @@ int xe_guc_submit_reset_prepare(struct xe_guc *guc) void xe_guc_submit_reset_wait(struct xe_guc *guc) { - wait_event(guc->ct.wq, xe_device_wedged(guc_to_xe(guc)) || + wait_event(guc->ct.wq, xe_wedged(guc_to_xe(guc)) || !xe_guc_read_stopped(guc)); } @@ -3307,7 +3313,7 @@ g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) struct xe_gt *gt = guc_to_gt(guc); struct xe_exec_queue *q; - if (unlikely(guc_id >= GUC_ID_MAX)) { + if (unlikely(guc_id >= xe_guc_id_mgr_max_usable(&guc->submission_state.idm))) { xe_gt_err(gt, "Invalid guc_id %u\n", guc_id); return NULL; } diff --git a/drivers/gpu/drm/xe/xe_hwmon.c b/drivers/gpu/drm/xe/xe_hwmon.c index 20c1cfffc5a03..defe8aaca7c11 100644 --- a/drivers/gpu/drm/xe/xe_hwmon.c +++ b/drivers/gpu/drm/xe/xe_hwmon.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -15,6 +16,7 @@ #include "regs/xe_pcode_regs.h" #include "xe_device.h" #include "xe_hwmon.h" +#include "xe_log.h" #include "xe_mmio.h" #include "xe_pcode.h" #include "xe_pcode_api.h" @@ -39,7 +41,11 @@ enum xe_hwmon_reg_operation { REG_READ64, }; -#define MAX_VRAM_CHANNELS (16) +#define MAX_VRAM_CHANNELS 80 +#define FIXED_VRAM_CHANNELS 16 + +/* Each MSU enable bit maps to one VRAM subsystem of 4 channels. */ +#define VRAM_CHANNELS_PER_MSU 4 enum xe_hwmon_channel { CHANNEL_CARD, @@ -48,6 +54,7 @@ enum xe_hwmon_channel { CHANNEL_MCTRL, CHANNEL_PCIE, CHANNEL_VRAM_N, + /* Compile-time upper bound; actual channel count is hwmon->temp.vram_count */ CHANNEL_VRAM_N_MAX = CHANNEL_VRAM_N + MAX_VRAM_CHANNELS - 1, CHANNEL_MAX, }; @@ -101,8 +108,15 @@ enum sensor_attr_power { #define PWR_ATTR_TO_STR(attr) (((attr) == hwmon_power_max) ? "PL1" : "PL2") -/* Index of memory controller in READ_THERMAL_DATA output */ +/* Sensor byte indices in READ_THERMAL_DATA output for BMG. */ #define TEMP_INDEX_MCTRL 2 +#define TEMP_INDEX_PCIE 10 +#define BMG_G21_TEMP_INDEX_PCIE 8 + +/* Sensor byte indices in READ_THERMAL_DATA output for CRI. */ +#define CRI_TEMP_INDEX_MCTRL 16 +#define CRI_TEMP_MCTRL_COUNT 4 +#define CRI_TEMP_INDEX_PCIE 20 /* Mailbox sentinel reported for an unavailable temperature sensor */ #define TEMP_MBX_INVALID 0xFF @@ -144,16 +158,22 @@ struct xe_hwmon_thermal_info { /** @data: temperature limits as little-endian dwords */ __le32 data[DIV_ROUND_UP(TEMP_LIMIT_MAX, sizeof(u32))]; }; - /** @count: no of temperature sensors available for the platform */ + /** @count: temperature sensors count from READ_THERMAL_CONFIG */ u8 count; + /** @vram_count: exclusive upper bound for VRAM temperature channel indices */ + u8 vram_count; + /** @available: temperature sensor availability cached at registration */ + bool available[CHANNEL_MAX]; + /** @msu_mask: VRAM subsystem enable mask read from MMIO */ + u32 msu_mask; union { /** @value: per-sensor raw mailbox temperature; bit7=sign, bits6:0=magnitude */ u8 value[U8_MAX + 1]; /** @dword: sensor values as little-endian dwords, u32-aligned for pcode reads */ __le32 dword[DIV_ROUND_UP(U8_MAX + 1, sizeof(u32))]; }; - /** @vram_label: vram label names */ - char vram_label[MAX_VRAM_CHANNELS][MAX_LABEL_SIZE]; + /** @vram_label: vram label names, dynamically allocated based on vram_count */ + char (*vram_label)[MAX_LABEL_SIZE]; }; /** @@ -296,6 +316,14 @@ static int xe_hwmon_pcode_set_default_power_limits(struct xe_hwmon *hwmon) return ret; } +static bool xe_hwmon_vram_channel_enabled(const struct xe_hwmon *hwmon, int index) +{ + if (hwmon->xe->info.platform == XE_CRESCENTISLAND) + return hwmon->temp.msu_mask & BIT(index / VRAM_CHANNELS_PER_MSU); + + return true; +} + static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg hwmon_reg, int channel) { @@ -303,12 +331,21 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg switch (hwmon_reg) { case REG_TEMP: - if (xe->info.platform == XE_BATTLEMAGE || xe->info.platform == XE_CRESCENTISLAND) { + if (xe->info.platform == XE_CRESCENTISLAND) { + if (channel == CHANNEL_PKG) + return CRI_PACKAGE_TEMPERATURE; + else if (channel == CHANNEL_VRAM) + return CRI_VRAM_TEMPERATURE; + else if (in_range(channel, CHANNEL_VRAM_N, hwmon->temp.vram_count) && + xe_hwmon_vram_channel_enabled(hwmon, channel - CHANNEL_VRAM_N)) + return CRI_VRAM_TEMPERATURE_N(channel - CHANNEL_VRAM_N); + } else if (xe->info.platform == XE_BATTLEMAGE) { if (channel == CHANNEL_PKG) return BMG_PACKAGE_TEMPERATURE; else if (channel == CHANNEL_VRAM) return BMG_VRAM_TEMPERATURE; - else if (in_range(channel, CHANNEL_VRAM_N, MAX_VRAM_CHANNELS)) + else if (in_range(channel, CHANNEL_VRAM_N, hwmon->temp.vram_count) && + xe_hwmon_vram_channel_enabled(hwmon, channel - CHANNEL_VRAM_N)) return BMG_VRAM_TEMPERATURE_N(channel - CHANNEL_VRAM_N); } else if (xe->info.platform == XE_DG2) { if (channel == CHANNEL_PKG) @@ -375,17 +412,20 @@ static struct xe_reg xe_hwmon_get_reg(struct xe_hwmon *hwmon, enum xe_hwmon_reg * same pattern for sysfs, allow arbitrary PL1 limits to be set but display * clamped values when read. */ -static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *value) +static int xe_hwmon_power_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *value) { u32 reg_val = 0; struct xe_device *xe = hwmon->xe; struct xe_reg rapl_limit, pkg_power_sku; struct xe_mmio *mmio = xe_root_tile_mmio(xe); + int ret = 0; mutex_lock(&hwmon->hwmon_lock); if (hwmon->xe->info.has_mbx_power_limits) { - xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + ret = xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + if (ret) + goto unlock; } else { rapl_limit = xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, channel); pkg_power_sku = xe_hwmon_get_reg(hwmon, REG_PKG_POWER_SKU, channel); @@ -417,6 +457,7 @@ static void xe_hwmon_power_max_read(struct xe_hwmon *hwmon, u32 attr, int channe } unlock: mutex_unlock(&hwmon->hwmon_lock); + return ret; } static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channel, long value) @@ -439,7 +480,9 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channe ret = xe_hwmon_pcode_rmw_power_limit(hwmon, attr, channel, PWR_LIM_EN, 0); if (ret) goto unlock; - xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + ret = xe_hwmon_pcode_read_power_limit(hwmon, attr, channel, ®_val); + if (ret) + goto unlock; } else { reg_val = xe_mmio_rmw32(mmio, rapl_limit, PWR_LIM_EN, 0); reg_val = xe_mmio_read32(mmio, rapl_limit); @@ -496,15 +539,18 @@ static int xe_hwmon_power_max_write(struct xe_hwmon *hwmon, u32 attr, int channe return ret; } -static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, - long *value) +static int xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, u32 attr, int channel, + long *value) { struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); u32 reg_val; + int ret = 0; if (hwmon->xe->info.has_mbx_power_limits) { /* PL1 is rated max if supported. */ - xe_hwmon_pcode_read_power_limit(hwmon, PL1_HWMON_ATTR, channel, ®_val); + ret = xe_hwmon_pcode_read_power_limit(hwmon, PL1_HWMON_ATTR, channel, ®_val); + if (ret) + return ret; } else { /* * This sysfs file won't be visible if REG_PKG_POWER_SKU is invalid, so valid check @@ -518,6 +564,7 @@ static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, u32 attr, int reg_val = REG_FIELD_GET(PKG_TDP, reg_val); *value = mul_u64_u32_shr(reg_val, SF_POWER, hwmon->scl_shift_power); + return ret; } /* @@ -540,7 +587,7 @@ static void xe_hwmon_power_rated_max_read(struct xe_hwmon *hwmon, u32 attr, int * hwmon->scl_shift_energy of 14 bits we have 57 (63 - 20 + 14) bits before * energyN_input overflows. This at 1000 W is an overflow duration of 278 years. */ -static void +static int xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) { struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); @@ -551,7 +598,7 @@ xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) /* Energy is supported only for card and pkg */ if (channel > CHANNEL_PKG) { *energy = 0; - return; + return 0; } if (hwmon->xe->info.platform == XE_BATTLEMAGE) { @@ -563,7 +610,7 @@ xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) if (ret != sizeof(pmt_val)) { drm_warn(&hwmon->xe->drm, "energy read from pmt failed, ret %d\n", ret); *energy = 0; - return; + return ret < 0 ? ret : -EIO; } if (channel == CHANNEL_PKG) @@ -580,6 +627,7 @@ xe_hwmon_energy_get(struct xe_hwmon *hwmon, int channel, long *energy) *energy = mul_u64_u32_shr(ei->accum_energy, SF_ENERGY, hwmon->scl_shift_energy); + return 0; } static ssize_t @@ -602,10 +650,11 @@ xe_hwmon_power_max_interval_show(struct device *dev, struct device_attribute *at if (hwmon->xe->info.has_mbx_power_limits) { ret = xe_hwmon_pcode_read_power_limit(hwmon, power_attr, channel, ®_val); if (ret) { - drm_err(&hwmon->xe->drm, - "power interval read fail, ch %d, attr %d, val 0x%08x, ret %d\n", - channel, power_attr, reg_val, ret); - reg_val = 0; + xe_log_err(hwmon->xe, HWMON, ret, + "Power interval read fail, ch %d, attr %d, val 0x%08x\n", + channel, power_attr, reg_val); + mutex_unlock(&hwmon->hwmon_lock); + return ret; } } else { reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_PKG_RAPL_LIMIT, @@ -797,30 +846,24 @@ static const struct attribute_group *hwmon_groups[] = { NULL }; +static const u32 hwmon_temp_config[CHANNEL_MAX + 1] = { + [CHANNEL_CARD] = HWMON_T_LABEL, + [CHANNEL_PKG] = HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | + HWMON_T_LABEL | HWMON_T_MAX, + [CHANNEL_VRAM ... CHANNEL_PCIE] = HWMON_T_CRIT | HWMON_T_EMERGENCY | + HWMON_T_INPUT | HWMON_T_LABEL, + [CHANNEL_VRAM_N ... CHANNEL_VRAM_N_MAX] = HWMON_T_CRIT | HWMON_T_EMERGENCY | + HWMON_T_INPUT | HWMON_T_LABEL, + [CHANNEL_MAX] = 0, /* hwmon config terminator */ +}; + +static const struct hwmon_channel_info hwmon_temp_info = { + .type = hwmon_temp, + .config = hwmon_temp_config, +}; + static const struct hwmon_channel_info * const hwmon_info[] = { - HWMON_CHANNEL_INFO(temp, - HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL | - HWMON_T_MAX, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL, - HWMON_T_CRIT | HWMON_T_EMERGENCY | HWMON_T_INPUT | HWMON_T_LABEL), + &hwmon_temp_info, HWMON_CHANNEL_INFO(power, HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_LABEL | HWMON_P_CRIT | HWMON_P_CAP, HWMON_P_MAX | HWMON_P_RATED_MAX | HWMON_P_LABEL | HWMON_P_CAP), @@ -834,9 +877,31 @@ static const struct hwmon_channel_info * const hwmon_info[] = { static int xe_hwmon_pcode_read_thermal_info(struct xe_hwmon *hwmon) { struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); + struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); u32 config = 0, data0 = 0, data1 = 0; int ret; + /* + * Only CRI reports dynamic VRAM channel state. Fixed-channel platforms + * set the count up front so those sensors stay visible even if the thermal + * mailbox reads below fail. + */ + if (hwmon->xe->info.has_fixed_vram_channels) + hwmon->temp.vram_count = FIXED_VRAM_CHANNELS; + + if (hwmon->xe->info.platform == XE_CRESCENTISLAND) { + hwmon->temp.msu_mask = xe_mmio_read32(mmio, CRI_MSU_VRAM_ENABLE); + hwmon->temp.vram_count = fls(hwmon->temp.msu_mask) * VRAM_CHANNELS_PER_MSU; + drm_dbg(&hwmon->xe->drm, "MSU VRAM enable mask 0x%x, VRAM channel bound %d\n", + hwmon->temp.msu_mask, hwmon->temp.vram_count); + if (hwmon->temp.vram_count > MAX_VRAM_CHANNELS) { + drm_warn(&hwmon->xe->drm, + "VRAM channel bound %d exceeds max %d, clamping\n", + hwmon->temp.vram_count, MAX_VRAM_CHANNELS); + hwmon->temp.vram_count = MAX_VRAM_CHANNELS; + } + } + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_LIMITS, 0), &data0, &data1); if (ret) @@ -889,17 +954,43 @@ static int mbx_temp_decode(u8 value) return (value & TEMP_MBX_SIGN) ? -temp : temp; } +static u8 xe_hwmon_pcie_temp_index(const struct xe_hwmon *hwmon) +{ + if (hwmon->xe->info.platform == XE_CRESCENTISLAND) + return CRI_TEMP_INDEX_PCIE; + + if (hwmon->xe->info.subplatform == XE_SUBPLATFORM_BATTLEMAGE_G21) + return BMG_G21_TEMP_INDEX_PCIE; + + return TEMP_INDEX_PCIE; +} + static int get_mc_temp(struct xe_hwmon *hwmon, long *val) { struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); - int ret, i, count = 0; + int ret, i, start, end, count = 0; s32 average = 0; + u32 data = 0; guard(mutex)(&hwmon->hwmon_lock); - for (i = 0; i < DIV_ROUND_UP(hwmon->temp.count, sizeof(u32)); i++) { - u32 data = 0; + /* CRI relocated the memory controller temperatures to READ_THERMAL_DATA bytes 16..19. */ + if (hwmon->xe->info.platform == XE_CRESCENTISLAND) { + start = CRI_TEMP_INDEX_MCTRL; + end = CRI_TEMP_INDEX_MCTRL + CRI_TEMP_MCTRL_COUNT; + } else { + start = TEMP_INDEX_MCTRL; + end = min_t(int, hwmon->temp.count - 1, xe_hwmon_pcie_temp_index(hwmon)); + } + if (end <= start || end > hwmon->temp.count) { + drm_warn(&hwmon->xe->drm, + "memory controller range [%d, %d) is invalid for temperature count %d\n", + start, end, hwmon->temp.count); + return -ENODATA; + } + + for (i = start / sizeof(u32); i < DIV_ROUND_UP(end, sizeof(u32)); i++) { ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_DATA, i), &data, NULL); if (ret) @@ -908,7 +999,7 @@ static int get_mc_temp(struct xe_hwmon *hwmon, long *val) hwmon->temp.dword[i] = cpu_to_le32(data); } - for (i = TEMP_INDEX_MCTRL; i < hwmon->temp.count - 1; i++) { + for (i = start; i < end; i++) { if (!mbx_temp_valid(hwmon, hwmon->temp.value[i])) continue; average += mbx_temp_decode(hwmon->temp.value[i]); @@ -928,23 +1019,36 @@ static int get_mc_temp(struct xe_hwmon *hwmon, long *val) return 0; } -static int get_pcie_temp(struct xe_hwmon *hwmon, long *val) +static int get_pcie_temp(const struct xe_hwmon *hwmon, long *val) { struct xe_tile *root_tile = xe_device_get_root_tile(hwmon->xe); u32 data = 0; + int index; int ret; + index = xe_hwmon_pcie_temp_index(hwmon); + if (index >= hwmon->temp.count) { + drm_warn(&hwmon->xe->drm, + "PCIe temperature index %d out of range for temperature count %d\n", + index, hwmon->temp.count); + return -ENODATA; + } + ret = xe_pcode_read(root_tile, PCODE_MBOX(PCODE_THERMAL_INFO, READ_THERMAL_DATA, - PCIE_SENSOR_GROUP_ID), &data, NULL); + index / sizeof(data)), + &data, NULL); if (ret) return ret; - /* Sensor offset is different for G21 */ - if (hwmon->xe->info.subplatform != XE_SUBPLATFORM_BATTLEMAGE_G21) - data = REG_FIELD_GET(PCIE_SENSOR_MASK, data); - + data >>= (index % sizeof(data)) * BITS_PER_BYTE; data = REG_FIELD_GET(TEMP_MASK, data); - *val = mbx_temp_decode(data) * MILLIDEGREE_PER_DEGREE; + if (!mbx_temp_valid(hwmon, data)) { + drm_dbg(&hwmon->xe->drm, "PCIe temperature not available\n"); + return -ENXIO; + } + + if (val) + *val = mbx_temp_decode(data) * MILLIDEGREE_PER_DEGREE; return 0; } @@ -1043,19 +1147,70 @@ static void xe_hwmon_get_voltage(struct xe_hwmon *hwmon, int channel, long *valu *value = DIV_ROUND_CLOSEST(REG_FIELD_GET(VOLTAGE_MASK, reg_val) * 2500, SF_VOLTAGE); } -static inline bool is_vram_ch_available(struct xe_hwmon *hwmon, int channel) +static bool xe_hwmon_temp_probe(struct xe_hwmon *hwmon, int channel) { struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); - int vram_id = channel - CHANNEL_VRAM_N; - struct xe_reg vram_reg; + struct xe_reg reg; + u32 reg_val; - vram_reg = xe_hwmon_get_reg(hwmon, REG_TEMP, channel); - if (!xe_reg_is_valid(vram_reg) || !xe_mmio_read32(mmio, vram_reg)) + switch (channel) { + case CHANNEL_PKG: + case CHANNEL_VRAM: + case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: + reg = xe_hwmon_get_reg(hwmon, REG_TEMP, channel); + if (!xe_reg_is_valid(reg)) + return false; + + /* + * On CRI, VRAM channel presence comes from the MSU enable mask + * (applied in xe_hwmon_get_reg()); the temperature register's + * 0xffffffff sentinel is not reliable for VRAM channels. + */ + if (hwmon->xe->info.platform == XE_CRESCENTISLAND && channel >= CHANNEL_VRAM_N) + return true; + + reg_val = xe_mmio_read32(mmio, reg); + if (!mmio_temp_valid(hwmon, reg_val)) { + drm_dbg(&hwmon->xe->drm, + "channel %d temperature unavailable, val 0x%x\n", channel, reg_val); + return false; + } + + return true; + case CHANNEL_MCTRL: + return hwmon->temp.count && !get_mc_temp(hwmon, NULL); + case CHANNEL_PCIE: + return hwmon->temp.count && !get_pcie_temp(hwmon, NULL); + default: return false; + } +} - /* Create label only for available vram channel */ - sprintf(hwmon->temp.vram_label[vram_id], "vram_ch_%d", vram_id); - return true; +static int xe_hwmon_init_temp_info(struct xe_hwmon *hwmon) +{ + struct device *dev = hwmon->xe->drm.dev; + int channel; + + if (hwmon->xe->info.has_mbx_thermal_info && xe_hwmon_pcode_read_thermal_info(hwmon)) + drm_warn(&hwmon->xe->drm, "Thermal mailbox not supported by card firmware\n"); + + /* vram_count is known only after reading thermal info above. */ + if (hwmon->temp.vram_count) { + hwmon->temp.vram_label = devm_kcalloc(dev, hwmon->temp.vram_count, + MAX_LABEL_SIZE, GFP_KERNEL); + if (!hwmon->temp.vram_label) + return -ENOMEM; + } + + for (channel = 0; channel < CHANNEL_MAX; channel++) { + hwmon->temp.available[channel] = xe_hwmon_temp_probe(hwmon, channel); + if (hwmon->temp.available[channel] && hwmon->temp.vram_label && + in_range(channel, CHANNEL_VRAM_N, hwmon->temp.vram_count)) + snprintf(hwmon->temp.vram_label[channel - CHANNEL_VRAM_N], MAX_LABEL_SIZE, + "vram_ch_%d", channel - CHANNEL_VRAM_N); + } + + return 0; } static umode_t @@ -1065,14 +1220,13 @@ xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) case hwmon_temp_emergency: switch (channel) { case CHANNEL_PKG: - return hwmon->temp.limit[TEMP_LIMIT_PKG_SHUTDOWN] ? 0444 : 0; - case CHANNEL_VRAM: - return hwmon->temp.limit[TEMP_LIMIT_MEM_SHUTDOWN] ? 0444 : 0; case CHANNEL_MCTRL: case CHANNEL_PCIE: - return hwmon->temp.count ? 0444 : 0; + return (hwmon->temp.available[channel] && + hwmon->temp.limit[TEMP_LIMIT_PKG_SHUTDOWN]) ? 0444 : 0; + case CHANNEL_VRAM: case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: - return (is_vram_ch_available(hwmon, channel) && + return (hwmon->temp.available[channel] && hwmon->temp.limit[TEMP_LIMIT_MEM_SHUTDOWN]) ? 0444 : 0; default: return 0; @@ -1080,14 +1234,13 @@ xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) case hwmon_temp_crit: switch (channel) { case CHANNEL_PKG: - return hwmon->temp.limit[TEMP_LIMIT_PKG_CRIT] ? 0444 : 0; - case CHANNEL_VRAM: - return hwmon->temp.limit[TEMP_LIMIT_MEM_CRIT] ? 0444 : 0; case CHANNEL_MCTRL: case CHANNEL_PCIE: - return hwmon->temp.count ? 0444 : 0; + return (hwmon->temp.available[channel] && + hwmon->temp.limit[TEMP_LIMIT_PKG_CRIT]) ? 0444 : 0; + case CHANNEL_VRAM: case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: - return (is_vram_ch_available(hwmon, channel) && + return (hwmon->temp.available[channel] && hwmon->temp.limit[TEMP_LIMIT_MEM_CRIT]) ? 0444 : 0; default: return 0; @@ -1095,7 +1248,8 @@ xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) case hwmon_temp_max: switch (channel) { case CHANNEL_PKG: - return hwmon->temp.limit[TEMP_LIMIT_PKG_MAX] ? 0444 : 0; + return (hwmon->temp.available[channel] && + hwmon->temp.limit[TEMP_LIMIT_PKG_MAX]) ? 0444 : 0; default: return 0; } @@ -1104,13 +1258,10 @@ xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) switch (channel) { case CHANNEL_PKG: case CHANNEL_VRAM: - return xe_reg_is_valid(xe_hwmon_get_reg(hwmon, REG_TEMP, - channel)) ? 0444 : 0; case CHANNEL_MCTRL: case CHANNEL_PCIE: - return hwmon->temp.count ? 0444 : 0; case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: - return is_vram_ch_available(hwmon, channel) ? 0444 : 0; + return hwmon->temp.available[channel] ? 0444 : 0; default: return 0; } @@ -1119,6 +1270,70 @@ xe_hwmon_temp_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) } } +#define IEEE754_SIGN REG_BIT(31) +#define IEEE754_EXP_MASK REG_GENMASK(30, 23) +#define IEEE754_MANTISSA_MASK REG_GENMASK(22, 0) +#define IEEE754_EXP_BIAS 127 /* Exponent bias to obtain the actual exponent */ +#define IEEE754_MANTISSA_BITS 23 /* Mantissa width and implicit leading-1 bit position */ +#define IEEE754_EXP_SPECIAL 0xff /* reserved exponent value encoding infinity or NaN */ + +/** + * xe_hwmon_temperature_decode - Decode an IEEE-754 temperature register value + * @reg_val: The raw register value to decode + * @temperature: Pointer to store the decoded temperature in millidegrees Celsius. + * + * IEEE-754 single-precision format: + * + * 31 30 23 22 0 + * +--+-----------------------------+----------------------------+ + * | S| exponent | fraction | + * +--+-----------------------------+----------------------------+ + * + * value = (-1)^S * 1.fraction * 2^(exponent - 127) + * + * Return: 0 on success or an error if @reg_val is non-finite or cannot be + * represented in millidegrees Celsius. + */ +static int xe_hwmon_temperature_decode(u32 reg_val, long *temperature) +{ + u32 exponent = REG_FIELD_GET(IEEE754_EXP_MASK, reg_val); + u32 mantissa = REG_FIELD_GET(IEEE754_MANTISSA_MASK, reg_val); + s64 value; + int shift; + + /* Exponent 0xff encodes infinity or NaN. */ + if (exponent == IEEE754_EXP_SPECIAL) + return -EINVAL; + + /* Subnormal values are smaller than one millidegree Celsius. */ + if (!exponent) { + *temperature = 0; + return 0; + } + + value = (s64)(mantissa | BIT(IEEE754_MANTISSA_BITS)) * MILLIDEGREE_PER_DEGREE; + shift = exponent - (IEEE754_EXP_BIAS + IEEE754_MANTISSA_BITS); + if (shift >= 0) { + if (shift >= BITS_PER_LONG || value > (LONG_MAX >> shift)) + return -ERANGE; + value <<= shift; + } else if (shift > -64) { + value >>= -shift; + } else { + value = 0; + } + + /* Ensure the decoded value fits in a long. */ + if (overflows_type(value, *temperature)) + return -ERANGE; + + if (reg_val & IEEE754_SIGN) + value = -value; + + *temperature = value; + return 0; +} + static int xe_hwmon_temp_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) { @@ -1128,26 +1343,33 @@ xe_hwmon_temp_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) switch (attr) { case hwmon_temp_input: switch (channel) { - case CHANNEL_PKG: - case CHANNEL_VRAM: - reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_TEMP, channel)); - - /* HW register value is in degrees Celsius, convert to millidegrees. */ - *val = REG_FIELD_GET(TEMP_MASK, reg_val) * MILLIDEGREE_PER_DEGREE; - return 0; case CHANNEL_MCTRL: return get_mc_temp(hwmon, val); case CHANNEL_PCIE: return get_pcie_temp(hwmon, val); + case CHANNEL_PKG: + case CHANNEL_VRAM: case CHANNEL_VRAM_N...CHANNEL_VRAM_N_MAX: reg_val = xe_mmio_read32(mmio, xe_hwmon_get_reg(hwmon, REG_TEMP, channel)); - /* - * This temperature format is 24 bit [31:8] signed integer and 8 bit - * [7:0] fraction. - */ - *val = (s32)(REG_FIELD_GET(TEMP_MASK_VRAM_N, reg_val)) * - (REG_FIELD_GET(TEMP_SIGN_MASK, reg_val) ? -1 : 1) * - MILLIDEGREE_PER_DEGREE; + if (!mmio_temp_valid(hwmon, reg_val)) { + drm_dbg(&hwmon->xe->drm, + "channel %d temperature invalid, val 0x%llx\n", + channel, reg_val); + return -ENXIO; + } + if (hwmon->xe->info.platform >= XE_CRESCENTISLAND) { + return xe_hwmon_temperature_decode((u32)reg_val, val); + } else if (channel >= CHANNEL_VRAM_N) { + /* + * This temperature format is 24 bit [31:8] signed integer and 8 bit + * [7:0] fraction for platforms before CRI. + */ + *val = (s32)REG_FIELD_GET(TEMP_MASK_VRAM_N, reg_val) * + (REG_FIELD_GET(TEMP_SIGN_MASK, reg_val) ? -1 : 1) * + MILLIDEGREE_PER_DEGREE; + } else { + *val = REG_FIELD_GET(TEMP_MASK, reg_val) * MILLIDEGREE_PER_DEGREE; + } return 0; default: return -EOPNOTSUPP; @@ -1229,7 +1451,8 @@ xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) } case hwmon_power_crit: if (channel == CHANNEL_CARD) { - xe_hwmon_pcode_read_i1(hwmon, &uval); + if (xe_hwmon_pcode_read_i1(hwmon, &uval)) + return 0; return (uval & POWER_SETUP_I1_WATTS) ? 0644 : 0; } break; @@ -1248,7 +1471,8 @@ xe_hwmon_power_is_visible(struct xe_hwmon *hwmon, u32 attr, int channel) } } if ((!(uval & PWR_LIM_EN)) && channel == CHANNEL_CARD) { - xe_hwmon_pcode_read_i1(hwmon, &uval); + if (xe_hwmon_pcode_read_i1(hwmon, &uval)) + return 0; return (uval & POWER_SETUP_I1_WATTS) ? 0444 : 0; } return (uval) ? 0444 : 0; @@ -1264,11 +1488,9 @@ xe_hwmon_power_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) switch (attr) { case hwmon_power_max: case hwmon_power_cap: - xe_hwmon_power_max_read(hwmon, attr, channel, val); - return 0; + return xe_hwmon_power_max_read(hwmon, attr, channel, val); case hwmon_power_rated_max: - xe_hwmon_power_rated_max_read(hwmon, attr, channel, val); - return 0; + return xe_hwmon_power_rated_max_read(hwmon, attr, channel, val); case hwmon_power_crit: return xe_hwmon_power_curr_crit_read(hwmon, channel, val, SF_POWER); default: @@ -1385,8 +1607,7 @@ xe_hwmon_energy_read(struct xe_hwmon *hwmon, u32 attr, int channel, long *val) { switch (attr) { case hwmon_energy_input: - xe_hwmon_energy_get(hwmon, channel, val); - return 0; + return xe_hwmon_energy_get(hwmon, channel, val); default: return -EOPNOTSUPP; } @@ -1555,8 +1776,10 @@ static int xe_hwmon_read_label(struct device *dev, *str = "mctrl"; else if (channel == CHANNEL_PCIE) *str = "pcie"; - else if (in_range(channel, CHANNEL_VRAM_N, MAX_VRAM_CHANNELS)) + else if (in_range(channel, CHANNEL_VRAM_N, hwmon->temp.vram_count)) *str = hwmon->temp.vram_label[channel - CHANNEL_VRAM_N]; + else + return -EOPNOTSUPP; return 0; case hwmon_power: case hwmon_energy: @@ -1584,7 +1807,7 @@ static const struct hwmon_chip_info hwmon_chip_info = { .info = hwmon_info, }; -static void +static int xe_hwmon_get_preregistration_info(struct xe_hwmon *hwmon) { struct xe_mmio *mmio = xe_root_tile_mmio(hwmon->xe); @@ -1641,8 +1864,7 @@ xe_hwmon_get_preregistration_info(struct xe_hwmon *hwmon) if (xe_hwmon_is_visible(hwmon, hwmon_fan, hwmon_fan_input, channel)) xe_hwmon_fan_input_read(hwmon, channel, &fan_speed); - if (hwmon->xe->info.has_mbx_thermal_info && xe_hwmon_pcode_read_thermal_info(hwmon)) - drm_warn(&hwmon->xe->drm, "Thermal mailbox not supported by card firmware\n"); + return xe_hwmon_init_temp_info(hwmon); } int xe_hwmon_register(struct xe_device *xe) @@ -1671,7 +1893,9 @@ int xe_hwmon_register(struct xe_device *xe) hwmon->xe = xe; xe->hwmon = hwmon; - xe_hwmon_get_preregistration_info(hwmon); + ret = xe_hwmon_get_preregistration_info(hwmon); + if (ret) + goto err_null_hwmon; drm_dbg(&xe->drm, "Register xe hwmon interface\n"); @@ -1680,11 +1904,15 @@ int xe_hwmon_register(struct xe_device *xe) &hwmon_chip_info, hwmon_groups); if (IS_ERR(hwmon->hwmon_dev)) { - drm_err(&xe->drm, "Failed to register xe hwmon (%pe)\n", hwmon->hwmon_dev); - xe->hwmon = NULL; - return PTR_ERR(hwmon->hwmon_dev); + ret = PTR_ERR(hwmon->hwmon_dev); + xe_log_err(xe, HWMON, ret, "Failed to register xe hwmon\n"); + goto err_null_hwmon; } return 0; + +err_null_hwmon: + xe->hwmon = NULL; + return ret; } MODULE_IMPORT_NS("INTEL_PMT_TELEMETRY"); diff --git a/drivers/gpu/drm/xe/xe_late_bind_fw.c b/drivers/gpu/drm/xe/xe_late_bind_fw.c index 1bddfaf0fb9f0..355d7c841871d 100644 --- a/drivers/gpu/drm/xe/xe_late_bind_fw.c +++ b/drivers/gpu/drm/xe/xe_late_bind_fw.c @@ -17,6 +17,12 @@ #include "xe_pcode.h" #include "xe_pcode_api.h" #include "xe_pm.h" +#include "xe_sysctrl.h" + +/* + * TODO: Add MODULE_FIRMWARE() for xe/cri_ocode_.bin once the + * Ocode firmware binary is available upstream. + */ /* * The component should load quite quickly in most cases, but it could take @@ -33,10 +39,12 @@ static const u32 fw_id_to_type[] = { [XE_LB_FW_FAN_CONTROL] = INTEL_LB_TYPE_FAN_CONTROL, + [XE_LB_FW_OCODE] = INTEL_LB_TYPE_OCODE, }; static const char * const fw_id_to_name[] = { [XE_LB_FW_FAN_CONTROL] = "fan_control", + [XE_LB_FW_OCODE] = "ocode", }; static struct xe_device * @@ -104,6 +112,7 @@ static int parse_cpd_header(struct xe_late_bind_fw *lb_fw, manifest = data + offset; lb_fw->version = manifest->fw_version; + lb_fw->sdk_api_version = manifest->ocode_data.sdk_api_version; return 0; } @@ -179,6 +188,34 @@ static const char *xe_late_bind_parse_status(uint32_t status) return "Invalid Payload"; case INTEL_LB_STATUS_TIMEOUT: return "Timeout"; + case INTEL_LB_STATUS_BUFFER_TOO_SMALL: + return "Buffer too small"; + case INTEL_LB_STATUS_INTERNAL_ERROR: + return "Internal Error"; + case INTEL_LB_STATUS_INVALID_FPT_TABLE: + return "Invalid FPT Table"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_VERIFICATION_ERROR: + return "Signed Payload Verification Error"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_CPD: + return "Signed Payload Invalid CPD"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_FW_VERSION_MISMATCH: + return "Signed Payload FW Version Mismatch"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_MANIFEST: + return "Signed Payload Invalid Manifest"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_INVALID_HASH: + return "Signed Payload Invalid Hash"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_BINDING_TYPE_MISMATCH: + return "Signed Payload Binding type Mismatch"; + case INTEL_LB_STATUS_SIGNED_PAYLOAD_HANDLE_SVN_FAILED: + return "Signed Payload Handle SVN Failed"; + case INTEL_LB_STATUS_DESTINATION_MBOX_FAILURE: + return "Destination MBOX Failure"; + case INTEL_LB_STATUS_MISSING_LOADING_PATCH: + return "Missing Loading Patch"; + case INTEL_LB_STATUS_INVALID_COMMAND: + return "Invalid Command"; + case INTEL_LB_STATUS_INVALID_HECI_HEADER: + return "Invalid HECI Header"; default: return "Unknown error"; } @@ -193,7 +230,7 @@ static int xe_late_bind_fw_num_fans(struct xe_late_bind *late_bind, u32 *num_fan PCODE_MBOX(FAN_SPEED_CONTROL, FSC_READ_NUM_FANS, 0), num_fans, NULL); } -void xe_late_bind_wait_for_worker_completion(struct xe_late_bind *late_bind) +static void wait_for_worker_completion(struct xe_late_bind *late_bind) { struct xe_device *xe = late_bind_to_xe(late_bind); struct xe_late_bind_fw *lbfw; @@ -269,18 +306,12 @@ static void xe_late_bind_work(struct work_struct *work) xe_pm_runtime_put(xe); } -int xe_late_bind_fw_load(struct xe_late_bind *late_bind) +static void xe_late_bind_fw_load(struct xe_late_bind *late_bind) { struct xe_device *xe = late_bind_to_xe(late_bind); struct xe_late_bind_fw *lbfw; int fw_id; - if (!late_bind->component_added) - return -ENODEV; - - if (late_bind->disable) - return 0; - for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) { lbfw = &late_bind->late_bind_fw[fw_id]; if (lbfw->payload) { @@ -288,7 +319,6 @@ int xe_late_bind_fw_load(struct xe_late_bind *late_bind) queue_work(late_bind->wq, &lbfw->work); } } - return 0; } static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) @@ -298,6 +328,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) struct xe_late_bind_fw *lb_fw; const struct firmware *fw; u32 num_fans; + u16 sdk_ver = 0; int ret; if (fw_id >= XE_LB_FW_MAX_ID) @@ -320,9 +351,23 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) return 0; } - snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), "xe/%s_8086_%04x_%04x_%04x.bin", - fw_id_to_name[lb_fw->id], pdev->device, - pdev->subsystem_vendor, pdev->subsystem_device); + if (lb_fw->type == INTEL_LB_TYPE_OCODE) { + /* Get SDK API version */ + ret = xe_sysctrl_sdk_api_version(xe, &sdk_ver); + if (ret) { + drm_warn(&xe->drm, "Failed to get OCODE SDK API version: %pe\n", + ERR_PTR(ret)); + return 0; + } + + snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), + "xe/cri_ocode_%u.bin", sdk_ver); + } else { + snprintf(lb_fw->blob_path, sizeof(lb_fw->blob_path), + "xe/%s_8086_%04x_%04x_%04x.bin", + fw_id_to_name[lb_fw->id], pdev->device, + pdev->subsystem_vendor, pdev->subsystem_device); + } drm_dbg(&xe->drm, "Request late binding firmware %s\n", lb_fw->blob_path); ret = firmware_request_nowarn(&fw, lb_fw->blob_path, xe->drm.dev); @@ -332,17 +377,17 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) return 0; } - if (fw->size > XE_LB_MAX_PAYLOAD_SIZE) { - drm_err(&xe->drm, "Firmware %s size %zu is larger than max pay load size %u\n", - lb_fw->blob_path, fw->size, XE_LB_MAX_PAYLOAD_SIZE); - ret = -ENODATA; - goto release_fw; - } - ret = parse_lb_layout(lb_fw, fw->data, fw->size, "LTES"); if (ret) goto release_fw; + if (lb_fw->type == INTEL_LB_TYPE_OCODE && lb_fw->sdk_api_version != sdk_ver) { + drm_warn(&xe->drm, "%s late binding fw: SDK API version mismatch %u != %u\n", + fw_id_to_name[lb_fw->id], lb_fw->sdk_api_version, sdk_ver); + ret = 0; + goto release_fw; + } + lb_fw->payload_size = fw->size; lb_fw->payload = drmm_kzalloc(&xe->drm, lb_fw->payload_size, GFP_KERNEL); if (!lb_fw->payload) { @@ -355,6 +400,10 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) lb_fw->version.major, lb_fw->version.minor, lb_fw->version.hotfix, lb_fw->version.build); + if (lb_fw->type == INTEL_LB_TYPE_OCODE) + drm_info(&xe->drm, "%s firmware SDK API version %u\n", + fw_id_to_name[lb_fw->id], lb_fw->sdk_api_version); + memcpy((void *)lb_fw->payload, fw->data, lb_fw->payload_size); INIT_WORK(&lb_fw->work, xe_late_bind_work); @@ -365,6 +414,7 @@ static int __xe_late_bind_fw_init(struct xe_late_bind *late_bind, u32 fw_id) static int xe_late_bind_fw_init(struct xe_late_bind *late_bind) { + struct xe_device *xe = late_bind_to_xe(late_bind); int ret; int fw_id; @@ -373,6 +423,8 @@ static int xe_late_bind_fw_init(struct xe_late_bind *late_bind) return -ENOMEM; for (fw_id = 0; fw_id < XE_LB_FW_MAX_ID; fw_id++) { + if (!(xe->info.late_bind_mask & BIT(fw_id))) + continue; ret = __xe_late_bind_fw_init(late_bind, fw_id); if (ret) return ret; @@ -399,7 +451,7 @@ static void xe_late_bind_component_unbind(struct device *xe_kdev, struct xe_device *xe = kdev_to_xe_device(xe_kdev); struct xe_late_bind *late_bind = &xe->late_bind; - xe_late_bind_wait_for_worker_completion(late_bind); + wait_for_worker_completion(late_bind); late_bind->component.ops = NULL; } @@ -414,7 +466,7 @@ static void xe_late_bind_remove(void *arg) struct xe_late_bind *late_bind = arg; struct xe_device *xe = late_bind_to_xe(late_bind); - xe_late_bind_wait_for_worker_completion(late_bind); + wait_for_worker_completion(late_bind); late_bind->component_added = false; @@ -425,6 +477,43 @@ static void xe_late_bind_remove(void *arg) } } +static bool late_bind_mei_enabled(struct xe_device *xe) +{ + if (!IS_ENABLED(CONFIG_INTEL_MEI_LB)) + return false; + + /* MEI exposed as xe auxiliary device */ + if (xe->info.has_heci_cscfi) + return IS_ENABLED(CONFIG_INTEL_MEI_GSC); + + /* Standalone MEI PCI function */ + return IS_ENABLED(CONFIG_INTEL_MEI_CSC); +} + +/** + * xe_late_bind_pm_suspend() - Wait for pending late binding fw loads + * @late_bind: pointer to late bind structure. + */ +void xe_late_bind_pm_suspend(struct xe_late_bind *late_bind) +{ + if (!late_bind->component_added) + return; + + wait_for_worker_completion(late_bind); +} + +/** + * xe_late_bind_pm_resume() - Reload late binding fw after resume + * @late_bind: pointer to late bind structure. + */ +void xe_late_bind_pm_resume(struct xe_late_bind *late_bind) +{ + if (!late_bind->component_added || late_bind->disable) + return; + + xe_late_bind_fw_load(late_bind); +} + /** * xe_late_bind_init() - add xe mei late binding component * @late_bind: pointer to late bind structure. @@ -436,10 +525,10 @@ int xe_late_bind_init(struct xe_late_bind *late_bind) struct xe_device *xe = late_bind_to_xe(late_bind); int err; - if (!xe->info.has_late_bind) + if (!xe->info.late_bind_mask) return 0; - if (!IS_ENABLED(CONFIG_INTEL_MEI_LB) || !IS_ENABLED(CONFIG_INTEL_MEI_GSC)) { + if (!late_bind_mei_enabled(xe)) { drm_info(&xe->drm, "Can't init xe mei late bind missing mei component\n"); return 0; } @@ -461,5 +550,7 @@ int xe_late_bind_init(struct xe_late_bind *late_bind) if (err) return err; - return xe_late_bind_fw_load(late_bind); + xe_late_bind_fw_load(late_bind); + + return 0; } diff --git a/drivers/gpu/drm/xe/xe_mert.c b/drivers/gpu/drm/xe/xe_mert.c index 934be933d57f7..a2eae6088c1ac 100644 --- a/drivers/gpu/drm/xe/xe_mert.c +++ b/drivers/gpu/drm/xe/xe_mert.c @@ -11,6 +11,7 @@ #include "xe_mmio.h" #include "xe_sriov_printk.h" #include "xe_tile.h" +#include "xe_wedge.h" /** * xe_mert_init_early() - Initialize MERT data @@ -74,8 +75,7 @@ static void mert_handle_cat_error(struct xe_device *xe) break; case CATERR_UNMAPPED_GGTT: xe_sriov_err(xe, "MERT: CAT_ERR: Access to an unmapped GGTT!\n"); - xe_device_declare_wedged(xe, - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); break; case CATERR_LMTT_FAULT: xe_sriov_dbg_ratelimited(xe, "MERT: CAT_ERR: VF%u LMTT fault!\n", vfid); @@ -83,8 +83,7 @@ static void mert_handle_cat_error(struct xe_device *xe) break; default: xe_sriov_err(xe, "MERT: Unexpected CAT_ERR code=%#x!\n", code); - xe_device_declare_wedged(xe, - DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_REBIND | DRM_WEDGE_RECOVERY_BUS_RESET); break; } } diff --git a/drivers/gpu/drm/xe/xe_pat.c b/drivers/gpu/drm/xe/xe_pat.c index 0ac64a6a3098a..5938fef22579a 100644 --- a/drivers/gpu/drm/xe/xe_pat.c +++ b/drivers/gpu/drm/xe/xe_pat.c @@ -16,6 +16,7 @@ #include "xe_force_wake.h" #include "xe_gt.h" #include "xe_gt_mcr.h" +#include "xe_log.h" #include "xe_mmio.h" #include "xe_sriov.h" #include "xe_wa.h" @@ -93,29 +94,29 @@ struct xe_pat_ops { }; static const struct xe_pat_table_entry xelp_pat_table[] = { - [0] = { XELP_PAT_WB, XE_COH_1WAY }, - [1] = { XELP_PAT_WC, XE_COH_NONE }, - [2] = { XELP_PAT_WT, XE_COH_NONE }, - [3] = { XELP_PAT_UC, XE_COH_NONE }, + [0] = { XELP_PAT_WB, XE_COH_1WAY, 1 }, + [1] = { XELP_PAT_WC, XE_COH_NONE, 1 }, + [2] = { XELP_PAT_WT, XE_COH_NONE, 1 }, + [3] = { XELP_PAT_UC, XE_COH_NONE, 1 }, }; static const struct xe_pat_table_entry xehpc_pat_table[] = { - [0] = { XELP_PAT_UC, XE_COH_NONE }, - [1] = { XELP_PAT_WC, XE_COH_NONE }, - [2] = { XELP_PAT_WT, XE_COH_NONE }, - [3] = { XELP_PAT_WB, XE_COH_1WAY }, - [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE }, - [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_1WAY }, - [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE }, - [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_1WAY }, + [0] = { XELP_PAT_UC, XE_COH_NONE, 1 }, + [1] = { XELP_PAT_WC, XE_COH_NONE, 1 }, + [2] = { XELP_PAT_WT, XE_COH_NONE, 1 }, + [3] = { XELP_PAT_WB, XE_COH_1WAY, 1 }, + [4] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WT, XE_COH_NONE, 1 }, + [5] = { XEHPC_PAT_CLOS(1) | XELP_PAT_WB, XE_COH_1WAY, 1 }, + [6] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WT, XE_COH_NONE, 1 }, + [7] = { XEHPC_PAT_CLOS(2) | XELP_PAT_WB, XE_COH_1WAY, 1 }, }; static const struct xe_pat_table_entry xelpg_pat_table[] = { - [0] = { XELPG_PAT_0_WB, XE_COH_NONE }, - [1] = { XELPG_PAT_1_WT, XE_COH_NONE }, - [2] = { XELPG_PAT_3_UC, XE_COH_NONE }, - [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_1WAY }, - [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_2WAY }, + [0] = { XELPG_PAT_0_WB, XE_COH_NONE, 1 }, + [1] = { XELPG_PAT_1_WT, XE_COH_NONE, 1 }, + [2] = { XELPG_PAT_3_UC, XE_COH_NONE, 1 }, + [3] = { XELPG_PAT_0_WB | XELPG_2_COH_1W, XE_COH_1WAY, 1 }, + [4] = { XELPG_PAT_0_WB | XELPG_3_COH_2W, XE_COH_2WAY, 1 }, }; /* @@ -294,25 +295,69 @@ static const struct xe_pat_table_entry xe3p_lpg_pat_table[] = { [31] = XE2_PAT( 0, 0, 3, 0, 3, 3 ), }; +bool xe_pat_index_is_valid(struct xe_device *xe, u16 pat_index) +{ + if (WARN_ON(pat_index >= xe->pat.n_entries)) + return false; + + return xe->pat.table[pat_index].valid; +} + u16 xe_pat_index_get_coh_mode(struct xe_device *xe, u16 pat_index) { - WARN_ON(pat_index >= xe->pat.n_entries); + WARN_ON(!xe_pat_index_is_valid(xe, pat_index)); + WARN_ON(!xe->pat.table[pat_index].coh_mode || + xe->pat.table[pat_index].coh_mode > XE_COH_2WAY); return xe->pat.table[pat_index].coh_mode; } bool xe_pat_index_get_comp_en(struct xe_device *xe, u16 pat_index) { - WARN_ON(pat_index >= xe->pat.n_entries); + WARN_ON(!xe_pat_index_is_valid(xe, pat_index)); return !!(xe->pat.table[pat_index].value & XE2_COMP_EN); } u16 xe_pat_index_get_l3_policy(struct xe_device *xe, u16 pat_index) { - WARN_ON(pat_index >= xe->pat.n_entries); + WARN_ON(!xe_pat_index_is_valid(xe, pat_index)); return REG_FIELD_GET(XE2_L3_POLICY, xe->pat.table[pat_index].value); } +bool xe_pat_wa_14026539277_reserved(struct xe_device *xe, u16 pat_index) +{ + struct xe_gt *gt; + u8 id; + bool has_wa = false; + + for_each_gt(gt, xe, id) { + if (XE_GT_WA(gt, 14026539277)) { + has_wa = true; + break; + } + } + + /* + * Disallow 2-way coherency for any L3 cache policy because + * XE_L3_POLICY_UC could still be promoted to a cached policy + * by MOCS indices. + */ + return has_wa && xe_pat_index_get_coh_mode(xe, pat_index) == XE_COH_2WAY; +} + +static u32 wa_14026539277_fixup_pat_value(struct xe_gt *gt, u32 value) +{ + if (XE_GT_WA(gt, 14026539277)) { + if (REG_FIELD_GET(XE2_COH_MODE, value) == XE_COH_2WAY) { + value &= ~(XE2_L3_POLICY | XE2_COH_MODE); + value |= REG_FIELD_PREP(XE2_L3_POLICY, XE_L3_POLICY_UC) | + REG_FIELD_PREP(XE2_COH_MODE, XE_COH_1WAY); + } + } + + return value; +} + static const struct xe_pat_table_entry *gt_pta_entry(struct xe_gt *gt) { struct xe_device *xe = gt_to_xe(gt); @@ -373,18 +418,22 @@ static void program_pat_mcr(struct xe_gt *gt, const struct xe_pat_table_entry ta for (int i = 0; i < n_entries; i++) { struct xe_reg_mcr reg_mcr = XE_REG_MCR(_PAT_INDEX(i)); + u32 pat = wa_14026539277_fixup_pat_value(gt, table[i].value); - xe_gt_mcr_multicast_write(gt, reg_mcr, table[i].value); + xe_gt_mcr_multicast_write(gt, reg_mcr, pat); } if (xe->pat.pat_ats) - xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), xe->pat.pat_ats->value); + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_ATS), + wa_14026539277_fixup_pat_value(gt, xe->pat.pat_ats->value)); if (pta_entry) - xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), pta_entry->value); + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_PTA), + wa_14026539277_fixup_pat_value(gt, pta_entry->value)); if (tr_pta_entry) - xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_TR_PTA), tr_pta_entry->value); + xe_gt_mcr_multicast_write(gt, XE_REG_MCR(_PAT_TR_PTA), + wa_14026539277_fixup_pat_value(gt, tr_pta_entry->value)); } static int xelp_dump(struct xe_gt *gt, struct drm_printer *p) @@ -534,13 +583,16 @@ static int xe2_dump(struct xe_gt *gt, struct drm_printer *p) drm_printf(p, "PAT table: (* = reserved entry)\n"); for (i = 0; i < xe->pat.n_entries; i++) { + bool rsvd = !xe->pat.table[i].valid || + xe_pat_wa_14026539277_reserved(xe, i); + if (xe_gt_is_media_type(gt)) pat = xe_mmio_read32(>->mmio, XE_REG(_PAT_INDEX(i))); else pat = xe_gt_mcr_unicast_read_any(gt, XE_REG_MCR(_PAT_INDEX(i))); xe_pat_index_label(label, sizeof(label), i); - xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); + xe->pat.ops->entry_dump(p, label, pat, rsvd); } /* @@ -692,8 +744,9 @@ int xe_pat_init_early(struct xe_device *xe) * raise an error rather than trying to silently inherit the * most recent platform's behavior. */ - drm_err(&xe->drm, "Missing PAT table for platform with graphics version %d.%02d!\n", - GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); + xe_log_err(xe, PAT, -ENODATA, + "Missing PAT table for platform with graphics version %d.%02d!\n", + GRAPHICS_VER(xe), GRAPHICS_VERx100(xe) % 100); return -ENODATA; } @@ -756,9 +809,14 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) for (u32 i = 0; i < xe->pat.n_entries; i++) { u32 pat = xe->pat.table[i].value; + pat = wa_14026539277_fixup_pat_value(gt, pat); + if (GRAPHICS_VER(xe) >= 20) { + bool rsvd = !xe->pat.table[i].valid || + xe_pat_wa_14026539277_reserved(xe, i); + xe_pat_index_label(label, sizeof(label), i); - xe->pat.ops->entry_dump(p, label, pat, !xe->pat.table[i].valid); + xe->pat.ops->entry_dump(p, label, pat, rsvd); } else if (xe->info.platform == XE_METEORLAKE) { xelpg_pat_entry_dump(p, i, pat); } else if (xe->info.platform == XE_PVC) { @@ -773,6 +831,8 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) if (pta_entry) { u32 pat = pta_entry->value; + pat = wa_14026539277_fixup_pat_value(gt, pat); + drm_printf(p, "Page Table Access:\n"); xe->pat.ops->entry_dump(p, "PTA_MODE", pat, false); } @@ -780,6 +840,8 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) if (tr_pta_entry) { u32 pat = tr_pta_entry->value; + pat = wa_14026539277_fixup_pat_value(gt, pat); + drm_printf(p, "TRTT Page Table Access:\n"); xe->pat.ops->entry_dump(p, "TR_PTA_MODE", pat, false); } @@ -787,6 +849,8 @@ int xe_pat_dump_sw_config(struct xe_gt *gt, struct drm_printer *p) if (xe->pat.pat_ats) { u32 pat = xe->pat.pat_ats->value; + pat = wa_14026539277_fixup_pat_value(gt, pat); + drm_printf(p, "PCIe ATS/PASID:\n"); xe->pat.ops->entry_dump(p, "PAT_ATS ", pat, false); } diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 12ff5c9025478..7fba5295c33e6 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -417,16 +417,17 @@ static const struct xe_device_desc bmg_desc = { .dma_mask_size = 46, .has_display = true, .has_fan_control = true, + .has_fixed_vram_channels = true, .has_flat_ccs = 1, .has_mbx_power_limits = true, .has_mbx_thermal_info = true, .has_gsc_nvm = 1, .has_heci_cscfi = 1, .has_i2c = true, - .has_late_bind = true, .has_pre_prod_wa = 1, .has_soc_remapper_telem = true, .has_sriov = true, + .late_bind_mask = BIT(XE_LB_FW_FAN_CONTROL), .max_gt_per_tile = 2, MULTI_LRC_MASK, .needs_scratch = true, @@ -487,6 +488,8 @@ static const struct xe_device_desc cri_desc = { .has_sriov = true, .has_sysctrl = true, .has_temp_sentinel = true, + .late_bind_mask = BIT(XE_LB_FW_FAN_CONTROL) | + BIT(XE_LB_FW_OCODE), .max_gt_per_tile = 2, MULTI_LRC_MASK, .require_force_probe = true, @@ -722,8 +725,9 @@ static int handle_gmdid(struct xe_device *xe, *graphics_ip = find_graphics_ip(ver); if (!*graphics_ip) { - drm_err(&xe->drm, "Hardware reports unknown graphics version %u.%02u\n", - ver / 100, ver % 100); + xe_log_err(xe, REGS, -EINVAL, + "Hardware reports unknown graphics version %u.%02u\n", + ver / 100, ver % 100); } ret = read_gmdid(xe, GMDID_MEDIA, &ver, media_revid); @@ -736,8 +740,9 @@ static int handle_gmdid(struct xe_device *xe, *media_ip = find_media_ip(ver); if (!*media_ip) { - drm_err(&xe->drm, "Hardware reports unknown media version %u.%02u\n", - ver / 100, ver % 100); + xe_log_err(xe, REGS, -EINVAL, + "Hardware reports unknown media version %u.%02u\n", + ver / 100, ver % 100); } return 0; @@ -798,6 +803,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_device_uid = desc->has_device_uid; xe->info.has_drm_ras = desc->has_drm_ras; xe->info.has_fan_control = desc->has_fan_control; + xe->info.has_fixed_vram_channels = desc->has_fixed_vram_channels; /* runtime fusing may force flat_ccs to disabled later */ xe->info.has_flat_ccs = desc->has_flat_ccs; xe->info.has_mbx_power_limits = desc->has_mbx_power_limits; @@ -806,7 +812,6 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.has_heci_gscfi = desc->has_heci_gscfi; xe->info.has_heci_cscfi = desc->has_heci_cscfi; xe->info.has_i2c = desc->has_i2c; - xe->info.has_late_bind = desc->has_late_bind; xe->info.has_llc = desc->has_llc; xe->info.has_mert = desc->has_mert; xe->info.has_page_reclaim_hw_assist = desc->has_page_reclaim_hw_assist; @@ -824,6 +829,7 @@ static int xe_info_init_early(struct xe_device *xe, xe->info.needs_scratch = desc->needs_scratch; xe->info.needs_shared_vf_gt_wq = desc->needs_shared_vf_gt_wq; xe->info.multi_lrc_mask = desc->multi_lrc_mask; + xe->info.late_bind_mask = desc->late_bind_mask; xe->info.probe_display = IS_ENABLED(CONFIG_DRM_XE_DISPLAY) && xe_modparam.probe_display && diff --git a/drivers/gpu/drm/xe/xe_pci_error.c b/drivers/gpu/drm/xe/xe_pci_error.c index a5c8fd67738cf..9bf0da3a3d152 100644 --- a/drivers/gpu/drm/xe/xe_pci_error.c +++ b/drivers/gpu/drm/xe/xe_pci_error.c @@ -13,6 +13,7 @@ #include "xe_printk.h" #include "xe_ras.h" #include "xe_survivability_mode.h" +#include "xe_wedge.h" static void prepare_device_for_reset(struct pci_dev *pdev) { @@ -22,7 +23,7 @@ static void prepare_device_for_reset(struct pci_dev *pdev) /* * Wedge the device to prevent userspace access but do not send the uevent. - * xe_device_wedged_fini() releases runtime pm if wedged flag is set, so acquire a runtime + * xe_wedge_fini() releases runtime pm if wedged flag is set, so acquire a runtime * pm reference to avoid underflow. */ if (!atomic_xchg(&xe->wedged.flag, 1)) @@ -31,7 +32,7 @@ static void prepare_device_for_reset(struct pci_dev *pdev) xe_device_set_in_reset(xe); for_each_gt(gt, xe, id) { - xe_gt_declare_wedged(gt); + xe_gt_wedge_declare(gt); flush_work(>->wedge.worker); } @@ -63,7 +64,7 @@ static pci_ers_result_t xe_pci_error_detected(struct pci_dev *pdev, pci_channel_ return PCI_ERS_RESULT_DISCONNECT; /* If the device is already wedged or in survivability mode, do not attempt recovery */ - if (xe_survivability_mode_is_boot_enabled(xe) || xe_device_wedged(xe)) + if (xe_survivability_mode_is_boot_enabled(xe) || xe_wedged(xe)) return PCI_ERS_RESULT_DISCONNECT; switch (state) { @@ -88,7 +89,7 @@ static pci_ers_result_t xe_pci_error_mmio_enabled(struct pci_dev *pdev) /* User wants to debug the error, prevent reset */ if (xe->wedged.mode == XE_WEDGED_MODE_UPON_ANY_HANG_NO_RESET) { - xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_NONE); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_NONE); return PCI_ERS_RESULT_DISCONNECT; } diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index d95f0d128ed49..61d68c09189f7 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -186,7 +186,7 @@ int xe_pm_suspend(struct xe_device *xe) if (err) goto err; - xe_late_bind_wait_for_worker_completion(&xe->late_bind); + xe_late_bind_pm_suspend(&xe->late_bind); for_each_gt(gt, xe, id) xe_gt_suspend_prepare(gt); @@ -249,7 +249,7 @@ int xe_pm_resume(struct xe_device *xe) xe_wa_apply_tile_workarounds(tile); for_each_gt(gt, xe, id) - xe_gt_idle_disable_c6(gt); + xe_gt_resume_early(gt); xe_display_pm_resume_early(xe); @@ -290,7 +290,7 @@ int xe_pm_resume(struct xe_device *xe) if (IS_VF_CCS_READY(xe)) xe_sriov_vf_ccs_register_context(xe); - xe_late_bind_fw_load(&xe->late_bind); + xe_late_bind_pm_resume(&xe->late_bind); drm_dbg(&xe->drm, "Device resumed\n"); xe_pm_block_end_signalling(); @@ -707,7 +707,7 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_wa_apply_tile_workarounds(tile); for_each_gt(gt, xe, id) - xe_gt_idle_disable_c6(gt); + xe_gt_resume_early(gt); xe_display_pm_resume_early(xe); @@ -755,7 +755,7 @@ int xe_pm_runtime_resume(struct xe_device *xe) xe_sriov_vf_ccs_register_context(xe); if (xe->d3cold.allowed) - xe_late_bind_fw_load(&xe->late_bind); + xe_late_bind_pm_resume(&xe->late_bind); out: xe_rpm_lockmap_release(xe); diff --git a/drivers/gpu/drm/xe/xe_ras.c b/drivers/gpu/drm/xe/xe_ras.c index fe16a41060607..2c01c183d02b1 100644 --- a/drivers/gpu/drm/xe/xe_ras.c +++ b/drivers/gpu/drm/xe/xe_ras.c @@ -3,7 +3,12 @@ * Copyright © 2026 Intel Corporation */ +#include + +#include "xe_assert.h" +#include "xe_bo.h" #include "xe_configfs.h" +#include "xe_cper.h" #include "xe_debugfs.h" #include "xe_device.h" #include "xe_drm_ras.h" @@ -16,6 +21,11 @@ #include "xe_sysctrl_event_types.h" #include "xe_sysctrl_mailbox.h" #include "xe_sysctrl_mailbox_types.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_wedge.h" + +/* Any non-null entry marks the page as offlined by firmware */ +#define XE_RAS_PAGE_OFFLINED xa_mk_value(1) #define CORE_COMPUTE_UNCORR_TYPE GENMASK(26, 25) /* @@ -103,7 +113,8 @@ static const char * const gpu_health_states[] = { }; static_assert(ARRAY_SIZE(gpu_health_states) == XE_RAS_HEALTH_MAX); -static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value); +static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, + u32 *value, bool emit_cper); static u8 drm_to_xe_ras_severity(u8 severity) { @@ -129,6 +140,20 @@ static u8 xe_to_drm_ras_severity(u8 severity) } } +static u8 ras_sev_to_cper_sev(u8 severity) +{ + switch (severity) { + case XE_RAS_SEV_CORRECTABLE: + return CPER_SEV_CORRECTED; + case XE_RAS_SEV_UNCORRECTABLE: + return CPER_SEV_RECOVERABLE; + case XE_RAS_SEV_INFORMATIONAL: + return CPER_SEV_INFORMATIONAL; + default: + return CPER_SEV_RECOVERABLE; + } +} + static u8 drm_to_xe_ras_component(u8 component) { switch (component) { @@ -165,6 +190,24 @@ static u8 xe_to_drm_ras_component(u8 component) } } +static enum xe_sigid ras_comp_to_hw_sigid(u8 component) +{ + switch (component) { + case XE_RAS_COMP_DEVICE_MEMORY: + return XE_SIGID_DEVICE_MEMORY; + case XE_RAS_COMP_CORE_COMPUTE: + return XE_SIGID_CORE_COMPUTE; + case XE_RAS_COMP_PCIE: + return XE_SIGID_PCIE; + case XE_RAS_COMP_FABRIC: + return XE_SIGID_FABRIC; + case XE_RAS_COMP_SOC_INTERNAL: + return XE_SIGID_SOC_INTERNAL; + default: + return XE_SIGID_HW; + } +} + static int ras_status_to_errno(u32 status) { switch (status) { @@ -185,6 +228,12 @@ static int ras_status_to_errno(u32 status) } } +static u16 ras_comp_to_log_comp(u8 component) +{ + return FIELD_PREP(XE_LOG_COMPONENT_CLASS_MASK, XE_LOG_COMPONENT_CLASS_HARDWARE) | + FIELD_PREP(XE_LOG_COMPONENT_TYPE_MASK, component); +} + static inline const char *sev_to_str(u8 severity) { if (severity >= XE_RAS_SEV_MAX) @@ -201,6 +250,142 @@ static inline const char *comp_to_str(u8 component) return xe_ras_components[component]; } +static int send_page_offline_cmd(struct xe_device *xe, u64 page_address, + enum xe_ras_page_action action) +{ + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_page_offline_request request = {0}; + struct xe_ras_page_offline_response response = {0}; + struct xe_ras_state *state = &xe->ras.state; + size_t rlen; + int ret; + + if (!xe->info.has_sysctrl) + return 0; + + xe_assert(xe, action < XE_RAS_PAGE_ACTION_MAX); + + request.page_address = page_address; + request.action = action; + + if (action == XE_RAS_PAGE_ACTION_OFFLINE) + xe_log_err(xe, DEVICE_MEMORY, 0, "Requesting firmware to offline page 0x%llx\n", + page_address); + else + xe_log_err(xe, DEVICE_MEMORY, 0, "Requesting firmware to remove page 0x%llx from queue\n", + page_address); + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_PAGE_OFFLINE, + &request, sizeof(request), &response, sizeof(response)); + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_log_err(xe, SYSCTRL, ret, "failed to send page offline command\n"); + return ret; + } + + if (rlen != sizeof(response)) { + xe_log_err(xe, SYSCTRL, -EINVAL, + "unexpected page offline response length %zu (expected %zu)\n", + rlen, sizeof(response)); + return -EINVAL; + } + + ret = ras_status_to_errno(response.status); + if (ret) { + xe_log_err(xe, SYSCTRL, ret, "page offline command failed with status %u\n", + response.status); + return ret; + } + + if (action == XE_RAS_PAGE_ACTION_OFFLINE) { + ret = xa_err(xa_store(&state->offlined_pages, + page_address >> XE_PTE_SHIFT, + XE_RAS_PAGE_OFFLINED, GFP_KERNEL)); + if (ret) + xe_err(xe, "Failed to store offlined page 0x%llx: %d\n", page_address, + ret); + } + + return 0; +} + +static int handle_page_offline(struct xe_device *xe, u64 page_address, bool notify_fw) +{ + struct xe_ras_state *state = &xe->ras.state; + enum xe_ras_page_action action; + u64 addr; + int ret; + + if (!IS_ALIGNED(page_address, XE_PAGE_SIZE)) { + xe_log_err(xe, SYSCTRL, -EINVAL, "Unaligned physical page address: 0x%llx\n", + page_address); + return -EINVAL; + } + + /* User policy set to disable page offlining */ + if (xe_ras_get_disable_vram_page_offline(xe)) { + action = XE_RAS_PAGE_ACTION_REMOVE; + xe_log_err(xe, DEVICE_MEMORY, -EOPNOTSUPP, "User policy set to disable page offlining\n"); + goto notify; + } + + addr = ALIGN_DOWN(page_address, PAGE_SIZE); + + ret = xe_ttm_vram_handle_addr_fault(xe, addr); + + /* + * Handle return code from address fault handling function: + * 0: Page softofflined, remove from firmware queue + * -EIO: Address belongs to a critical BO/stolen area that cannot be offlined + * -ENOMEM: Failed to allocate BO for soft-offlining the page + * -EEXIST: Address is soft offlined but yet to be offlined by firmware for second + * occurrence + */ + + switch (ret) { + case 0: + action = XE_RAS_PAGE_ACTION_REMOVE; + xe_log_err(xe, DEVICE_MEMORY, ret, + "Poison detected at physical address 0x%llx, page soft-offlined\n", + page_address); + break; + case -EIO: + xe_log_err(xe, DEVICE_MEMORY, ret, + "Poison detected at physical address 0x%llx, page belongs to critical BO and cannot be soft-offlined\n", + page_address); + return ret; + case -EEXIST: + if (xa_load(&state->offlined_pages, page_address >> XE_PTE_SHIFT)) + action = XE_RAS_PAGE_ACTION_REMOVE; + else + action = XE_RAS_PAGE_ACTION_OFFLINE; + + xe_log_err(xe, DEVICE_MEMORY, ret, + "Double-bit ECC error detected at physical address 0x%llx, page soft-offlined\n", + page_address); + break; + case -ENOMEM: + xe_log_err(xe, DEVICE_MEMORY, ret, + "Poison detected at physical address 0x%llx, failed to soft-offline\n", + page_address); + return ret; + default: + xe_log_err(xe, DEVICE_MEMORY, ret, "Failed to handle address fault at physical address 0x%llx\n", + page_address); + return 0; + } + + notify: + if (notify_fw) { + ret = send_page_offline_cmd(xe, page_address, action); + if (ret) + return ret; + } + + return 0; +} + static bool ras_counter_is_valid(struct xe_device *xe, struct xe_ras_error_class *counter) { u8 severity = counter->common.severity; @@ -219,6 +404,106 @@ static bool ras_counter_is_valid(struct xe_device *xe, struct xe_ras_error_class return true; } +/* + * Fetch and process offline list and queue from firmware via mailbox. @notify_fw controls + * whether firmware is notified of the offline/remove action. + */ +static void process_offline_pages(struct xe_device *xe, u32 cmd, void *req, size_t req_size, + void *resp, size_t resp_size, + struct xe_ras_offline_common *common, bool notify_fw) +{ + struct xe_sysctrl_mailbox_command command = {0}; + struct xe_ras_offline_list_request *list_req; + struct xe_ras_state *state = &xe->ras.state; + unsigned long index; + u32 total_pages = 0, count = 0; + size_t rlen; + int ret, i; + + list_req = req; + + xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, cmd, req, req_size, resp, + resp_size); + + do { + memset(resp, 0, resp_size); + + if (list_req) + list_req->index = count; + + ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); + if (ret) { + xe_log_err(xe, SYSCTRL, ret, "failed to get page offline data, cmd=%#x\n", + cmd); + return; + } + + if (rlen != resp_size) { + xe_log_err(xe, SYSCTRL, -EINVAL, + "unexpected page offline response length %zu (expected %zu), cmd=%#x\n", + rlen, resp_size, cmd); + return; + } + + /* Populate the maximum number of pages from offline list */ + if (list_req && !state->max_pages) + state->max_pages = + ((struct xe_ras_offline_list_response *)resp)->max_entries; + + for (i = 0; i < common->pages_returned && i < XE_RAS_NUM_PAGES; i++) { + handle_page_offline(xe, common->page_addresses[i], notify_fw); + /* The pages are already offlined by firmware */ + if (!notify_fw) { + index = common->page_addresses[i] >> XE_PTE_SHIFT; + ret = xa_err(xa_store(&state->offlined_pages, index, + XE_RAS_PAGE_OFFLINED, GFP_KERNEL)); + if (ret) + xe_err(xe, "Failed to store offlined page 0x%llx: %d\n", + common->page_addresses[i], ret); + } + } + + count += common->pages_returned; + if (!common->pages_returned) + break; + + if (!total_pages) + total_pages = common->total_pages; + + if (count > total_pages) { + xe_log_err(xe, SYSCTRL, -EINVAL, + "Pages returned exceed total pages %u, returned %u, cmd=%#x\n", + total_pages, count, cmd); + return; + } + } while (common->additional_data); +} + +/* + * Retrieve firmware queued pages from warm-retained SRAM during probe. Handle and soft-offline + * each page as required. + */ +static void get_offline_queue(struct xe_device *xe) +{ + struct xe_ras_offline_common response = {0}; + + process_offline_pages(xe, XE_SYSCTRL_CMD_GET_OFFLINE_QUEUE, NULL, 0, &response, + sizeof(response), &response, true); +} + +/* + * Retrieve permanently offlined pages from firmware's non-volatile storage during probe, + * allowing the driver to soft-offline them before kernel allocations begin. + */ +static void get_offline_list(struct xe_device *xe) +{ + struct xe_ras_offline_list_response response = {0}; + struct xe_ras_offline_list_request request = {0}; + + process_offline_pages(xe, XE_SYSCTRL_CMD_GET_OFFLINE_LIST, &request, sizeof(request), + &response, sizeof(response), &response.common, false); +} + static struct pci_dev *find_usp_dev(struct pci_dev *pdev) { struct pci_dev *vsp; @@ -291,7 +576,7 @@ static void ras_send_error_event(struct xe_device *xe, u8 severity, u8 component counter.common.severity = severity; counter.common.component = component; - ret = get_counter(xe, &counter, &value); + ret = get_counter(xe, &counter, &value, false); if (ret) return; @@ -301,13 +586,17 @@ static void ras_send_error_event(struct xe_device *xe, u8 severity, u8 component xe_drm_ras_event(xe, drm_component, drm_severity, value); } -static u8 handle_core_compute_errors(struct xe_ras_error_array *arr) +static u8 handle_core_compute_errors(struct xe_device *xe, struct xe_ras_error_array *arr) { struct xe_ras_compute_error *error_info = (void *)arr->details; + u8 cper_sev = ras_sev_to_cper_sev(arr->counter.common.severity); u8 uncorr_type; uncorr_type = FIELD_GET(CORE_COMPUTE_UNCORR_TYPE, error_info->log_header); + xe_log_comp(xe, cper_sev, CORE_COMPUTE, &arr->counter, sizeof(arr->counter), + "Uncorrectable error detected, type %u\n", uncorr_type); + /* Request a reset if error is global */ if (uncorr_type == GLOBAL_UNCORR_ERROR) return XE_RAS_RECOVERY_ACTION_RESET; @@ -321,14 +610,16 @@ static u8 handle_core_compute_errors(struct xe_ras_error_array *arr) static void punit_error_handler(struct xe_device *xe) { - xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_COLD_RESET); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_COLD_RESET); } static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_array *arr) { struct xe_ras_soc_error *info = (void *)arr->details; struct xe_ras_soc_error_source *source = &info->source; - struct xe_ras_error_class *counter = &arr->counter; + u32 raw_source; + + BUILD_BUG_ON(sizeof(raw_source) != sizeof(*source)); if (source->csc) { struct xe_ras_csc_error *csc_error = (void *)info->details; @@ -342,9 +633,8 @@ static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_a * is required. */ if (csc_error->hec_fw_error) { - xe_err(xe, "[RAS]: CSC %s detected: 0x%x\n", - sev_to_str(counter->common.severity), - csc_error->hec_fw_error); + xe_log_comp_fatal(xe, SOC_INTERNAL, &arr->counter, sizeof(arr->counter), + "CSC error detected: 0x%08x\n", csc_error->hec_fw_error); xe_survivability_mode_runtime_enable(xe); return XE_RAS_RECOVERY_ACTION_DISCONNECT; } @@ -352,42 +642,56 @@ static u8 handle_soc_internal_errors(struct xe_device *xe, struct xe_ras_error_a struct xe_ras_ieh_error *ieh_error = (void *)info->details; if (ieh_error->global_error_status & XE_RAS_SOC_IEH_PUNIT) { - xe_err(xe, "[RAS]: PUNIT %s detected: 0x%x\n", - sev_to_str(counter->common.severity), - ieh_error->global_error_status); + xe_log_comp_fatal(xe, SOC_INTERNAL, &arr->counter, sizeof(arr->counter), + "PUNIT error detected: 0x%08x\n", + ieh_error->global_error_status); punit_error_handler(xe); return XE_RAS_RECOVERY_ACTION_DISCONNECT; } } /* For other SoC internal errors, request a reset as recovery mechanism */ + memcpy(&raw_source, source, sizeof(raw_source)); + xe_log_comp(xe, ras_sev_to_cper_sev(arr->counter.common.severity), SOC_INTERNAL, + &arr->counter, sizeof(arr->counter), + "Unknown error detected: 0x%08x\n", raw_source); return XE_RAS_RECOVERY_ACTION_RESET; } static u8 handle_device_memory_errors(struct xe_device *xe, struct xe_ras_error_array *arr) { struct xe_ras_memory_error *info = (void *)arr->details; + u8 cper_sev = ras_sev_to_cper_sev(arr->counter.common.severity); + int ret; /* * For memory errors, the recovery action depends on the error category * - * TODO: Double-bit ECC errors: Page offlining + * Double-bit ECC errors: Page offlining * Poison and data parity errors: Log only * For any other memory errors, request a reset as recovery mechanism */ switch (info->category) { case XE_RAS_MEMORY_POISON: - xe_info(xe, "[RAS]: Poison error detected\n"); + xe_log_comp_info(xe, DEVICE_MEMORY, &arr->counter, sizeof(arr->counter), + "Poison error detected\n"); break; case XE_RAS_MEMORY_DATA_PARITY: - xe_info(xe, "[RAS]: Data parity error detected\n"); + xe_log_comp_info(xe, DEVICE_MEMORY, &arr->counter, sizeof(arr->counter), + "Data parity error detected\n"); break; case XE_RAS_MEMORY_DB_ECC: - xe_info(xe, "[RAS]: Double-bit ECC error detected at sw address 0x%llx\n", - info->sw_address); - /* TODO: Add page offlining for Double-bit ECC error */ - fallthrough; + ret = handle_page_offline(xe, info->sw_address, true); + xe_log_comp(xe, ret ? cper_sev : CPER_SEV_INFORMATIONAL, + DEVICE_MEMORY, &arr->counter, sizeof(arr->counter), + "Double-bit ECC error detected at sw address 0x%llx\n", + info->sw_address); + if (ret) + return XE_RAS_RECOVERY_ACTION_RESET; + break; default: + xe_log_comp(xe, cper_sev, DEVICE_MEMORY, &arr->counter, sizeof(arr->counter), + "Unknown error detected\n"); return XE_RAS_RECOVERY_ACTION_RESET; } @@ -420,8 +724,11 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe, if (!ras_counter_is_valid(xe, &errors[id])) continue; - xe_warn(xe, "[RAS]: %s %s detected\n", - comp_to_str(component), sev_to_str(severity)); + xe_log_from(xe, ras_sev_to_cper_sev(severity), + ras_comp_to_hw_sigid(component), + ras_comp_to_log_comp(component), + &errors[id], sizeof(errors[id]), + "error detected\n"); /* Send event once per component */ if (sent & BIT(component)) @@ -432,9 +739,37 @@ void xe_ras_counter_threshold_crossed(struct xe_device *xe, } } -static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, u32 *value) +static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, + u32 *value, bool emit_cper) { struct xe_ras_get_counter_response response = {0}; + int ret; + + ret = xe_ras_get_counter_response(xe, counter, &response); + if (ret) + return ret; + + *value = response.value; + if (emit_cper) + xe_cper_emit_counter(xe, ras_sev_to_cper_sev(counter->common.severity), + ras_comp_to_hw_sigid(counter->common.component), + counter, &response); + return 0; +} + +/** + * xe_ras_get_counter_response() - Get error counter record + * @xe: Xe device instance + * @counter: RAS error class + * @response: Counter record retrieved + * + * Retrieve the counter record of a specific error class from firmware. + * + * Return: 0 on success, negative error code on failure. + */ +int xe_ras_get_counter_response(struct xe_device *xe, const struct xe_ras_error_class *counter, + struct xe_ras_get_counter_response *response) +{ struct xe_ras_get_counter_request request = {0}; struct xe_sysctrl_mailbox_command command = {0}; struct xe_ras_error_common *common; @@ -444,7 +779,7 @@ static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, request.counter = *counter; xe_sysctrl_create_command(&command, XE_SYSCTRL_GROUP_GFSP, XE_SYSCTRL_CMD_GET_COUNTER, - &request, sizeof(request), &response, sizeof(response)); + &request, sizeof(request), response, sizeof(*response)); ret = xe_sysctrl_send_command(&xe->sc, &command, &rlen); if (ret) { @@ -452,20 +787,19 @@ static int get_counter(struct xe_device *xe, struct xe_ras_error_class *counter, return ret; } - if (rlen != sizeof(response)) { + if (rlen != sizeof(*response)) { xe_err(xe, "sysctrl: unexpected get counter response length %zu (expected %zu)\n", - rlen, sizeof(response)); + rlen, sizeof(*response)); return -EIO; } - if (!ras_counter_is_valid(xe, &response.counter)) + if (!ras_counter_is_valid(xe, &response->counter)) return -EBADMSG; - common = &response.counter.common; - *value = response.value; + common = &response->counter.common; - xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", *value, comp_to_str(common->component), - sev_to_str(common->severity)); + xe_dbg(xe, "[RAS]: get counter %u for %s %s\n", response->value, + comp_to_str(common->component), sev_to_str(common->severity)); return 0; } @@ -547,7 +881,7 @@ enum xe_ras_recovery_action xe_ras_process_errors(struct xe_device *xe) switch (component) { case XE_RAS_COMP_CORE_COMPUTE: - action = handle_core_compute_errors(arr); + action = handle_core_compute_errors(xe, arr); break; case XE_RAS_COMP_SOC_INTERNAL: action = handle_soc_internal_errors(xe, arr); @@ -600,7 +934,7 @@ int xe_ras_get_counter(struct xe_device *xe, u8 severity, u8 component, u32 *val counter.common.component = drm_to_xe_ras_component(component); guard(xe_pm_runtime)(xe); - return get_counter(xe, &counter, value); + return get_counter(xe, &counter, value, READ_ONCE(xe->cper.on_query)); } /** @@ -905,33 +1239,83 @@ static const struct attribute_group gpu_health_group = { .attrs = gpu_health_attrs, }; +static void ras_fini(void *arg) +{ + struct xe_device *xe = arg; + + xa_destroy(&xe->ras.state.offlined_pages); +} + +/** + * xe_ras_get_disable_vram_page_offline - Get the vram page offline user policy + * @xe: xe device instance + * + * Return: true if the vram page offline policy is disabled, false otherwise. + */ +bool xe_ras_get_disable_vram_page_offline(struct xe_device *xe) +{ + return xe->ras.state.disable_vram_page_offline; +} + +/** + * xe_ras_get_max_pages - Get the maximum number of pages + * @xe: xe device instance + * + * Return: Maximum number of pages stored by the firmware in flash + */ +u32 xe_ras_get_max_pages(struct xe_device *xe) +{ + return xe->ras.state.max_pages; +} + /** * xe_ras_init - Initialize Xe RAS * @xe: xe device instance * * Initialize Xe RAS + * + * Return: 0 on success, negative error code otherwise */ -void xe_ras_init(struct xe_device *xe) +int xe_ras_init(struct xe_device *xe) { + struct xe_ras_state *state = &xe->ras.state; int ret; - /* - * TODO: Replace platform check with xe->info.has_disable_vram_page_offline - * once the feature flag is plumbed through device info. - */ - if (xe->info.platform == XE_CRESCENTISLAND) - xe->ras.disable_vram_page_offline = - xe_configfs_get_disable_vram_page_offline(to_pci_dev(xe->drm.dev)); - xe_drm_ras_init(xe); if (!xe->info.has_sysctrl) - return; + return 0; + + /* + * TODO: Replace with xe->info.has_disable_vram_page_offline once the feature flag is + * plumbed through device info. + */ + xe->ras.state.disable_vram_page_offline = + xe_configfs_get_disable_vram_page_offline(to_pci_dev(xe->drm.dev)); if (IS_ENABLED(CONFIG_PCIEAER)) ras_usp_aer_init(xe); + xa_init(&state->offlined_pages); + + ret = devm_add_action_or_reset(xe->drm.dev, ras_fini, xe); + if (ret) + return ret; + + /* Fetch the offline list before queue to avoid duplication */ + get_offline_list(xe); + get_offline_queue(xe); + + /* + * During probe, process and log any errors detected by firmware while the driver was not + * loaded. Critical errors such as Punit and CSC are reported through Pcode init failure, + * causing the driver to enter survivability mode. + */ + xe_ras_process_errors(xe); + ret = devm_device_add_group(xe->drm.dev, &gpu_health_group); if (ret) - xe_err(xe, "Failed to create GPU health sysfs, err=%d\n", ret); + return ret; + + return xe_cper_sysfs_init(xe); } diff --git a/drivers/gpu/drm/xe/xe_survivability_mode.c b/drivers/gpu/drm/xe/xe_survivability_mode.c index ca031576fecb7..d73cdad0b3a9c 100644 --- a/drivers/gpu/drm/xe/xe_survivability_mode.c +++ b/drivers/gpu/drm/xe/xe_survivability_mode.c @@ -20,6 +20,7 @@ #include "xe_pcode_api.h" #include "xe_printk.h" #include "xe_vsec.h" +#include "xe_wedge.h" /** * DOC: Survivability Mode @@ -446,7 +447,7 @@ void xe_survivability_mode_runtime_enable(struct xe_device *xe) survivability->type = XE_SURVIVABILITY_TYPE_RUNTIME; xe_log_err(xe, SURVIVABILITY, 0, "Runtime Mode enabled!\n"); - xe_device_declare_wedged(xe, DRM_WEDGE_RECOVERY_VENDOR); + xe_wedge_declare(xe, DRM_WEDGE_RECOVERY_VENDOR); xe_log_err(xe, SURVIVABILITY, 0, "Firmware flash required!\n"); xe_info(xe, "Please refer to the userspace documentation for more details how to flash the firmware on %s!\n",