Re: [smartpqi updates PATCH V2 02/11] smartpqi: add controller handshake during kdump

[john.p.donnelly@oracle.com]

On 9/28/21 6:54 PM, Don Brace wrote:
> From: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
> 
> Correct kdump hangs when controller is locked up.
> 
> There are occasions when a controller reboot
> (controller soft reset) is issued when a controller
> firmware crash dump is in progress.
> 
> This leads to incomplete controller firmware crash dump.
>   - When the controller crash dump is in progress,
>     and a kdump is initiated, the driver issues
>     inbound doorbell reset to bring back the
>     controller in SIS mode.
>   - If the controller is in locked up state,
>     the inbound doorbell reset does not work causing
>     controller initialization failures. This results
>     in the driver hanging waiting for SIS mode.
> 
> To avoid an incomplete controller crash dump, add in
> a controller crash dump handshake.
>   - Controller will indicate start and end of the controller
>     crash dump by setting some register bits.
>   - Driver will look these bits when a kdump is initiated.
>     If a controller crash dump is in progress, the driver will
>     wait for the controller crash dump to complete
>     before issuing the controller soft reset then complete
>     driver initialization.
> 
> Reviewed-by: Scott Benesh <scott.benesh@microchip.com>
> Reviewed-by: Scott Teel <scott.teel@microchip.com>
> Reviewed-by: Mike McGowen <mike.mcgowen@microchip.com>
> Signed-off-by: Mahesh Rajashekhara <mahesh.rajashekhara@microchip.com>
> Signed-off-by: Don Brace <don.brace@microchip.com>

Acked-by: John Donnelly <john.p.donnelly@oracle.com>


> ---
>   drivers/scsi/smartpqi/smartpqi_init.c | 41 +++++++++++++++++++--
>   drivers/scsi/smartpqi/smartpqi_sis.c  | 51 +++++++++++++++++++++++++++
>   drivers/scsi/smartpqi/smartpqi_sis.h  |  1 +
>   3 files changed, 91 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/scsi/smartpqi/smartpqi_init.c b/drivers/scsi/smartpqi/smartpqi_init.c
> index 97027574eb1f..5655d240f7a7 100644
> --- a/drivers/scsi/smartpqi/smartpqi_init.c
> +++ b/drivers/scsi/smartpqi/smartpqi_init.c
> @@ -234,15 +234,46 @@ static inline bool pqi_is_hba_lunid(u8 *scsi3addr)
>   	return pqi_scsi3addr_equal(scsi3addr, RAID_CTLR_LUNID);
>   }
>   
> +#define PQI_DRIVER_SCRATCH_PQI_MODE			0x1
> +#define PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED		0x2
> +
>   static inline enum pqi_ctrl_mode pqi_get_ctrl_mode(struct pqi_ctrl_info *ctrl_info)
>   {
> -	return sis_read_driver_scratch(ctrl_info);
> +	return sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_PQI_MODE ? PQI_MODE : SIS_MODE;
>   }
>   
>   static inline void pqi_save_ctrl_mode(struct pqi_ctrl_info *ctrl_info,
>   	enum pqi_ctrl_mode mode)
>   {
> -	sis_write_driver_scratch(ctrl_info, mode);
> +	u32 driver_scratch;
> +
> +	driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> +	if (mode == PQI_MODE)
> +		driver_scratch |= PQI_DRIVER_SCRATCH_PQI_MODE;
> +	else
> +		driver_scratch &= ~PQI_DRIVER_SCRATCH_PQI_MODE;
> +
> +	sis_write_driver_scratch(ctrl_info, driver_scratch);
> +}
> +
> +static inline bool pqi_is_fw_triage_supported(struct pqi_ctrl_info *ctrl_info)
> +{
> +	return (sis_read_driver_scratch(ctrl_info) & PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED) != 0;
> +}
> +
> +static inline void pqi_save_fw_triage_setting(struct pqi_ctrl_info *ctrl_info, bool is_supported)
> +{
> +	u32 driver_scratch;
> +
> +	driver_scratch = sis_read_driver_scratch(ctrl_info);
> +
> +	if (is_supported)
> +		driver_scratch |= PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> +	else
> +		driver_scratch &= ~PQI_DRIVER_SCRATCH_FW_TRIAGE_SUPPORTED;
> +
> +	sis_write_driver_scratch(ctrl_info, driver_scratch);
>   }
>   
>   static inline void pqi_ctrl_block_scan(struct pqi_ctrl_info *ctrl_info)
> @@ -7292,6 +7323,7 @@ static void pqi_ctrl_update_feature_flags(struct pqi_ctrl_info *ctrl_info,
>   		ctrl_info->unique_wwid_in_report_phys_lun_supported =
>   			firmware_feature->enabled;
>   		break;
> +		pqi_save_fw_triage_setting(ctrl_info, firmware_feature->enabled);
>   	}
>   
>   	pqi_firmware_feature_status(ctrl_info, firmware_feature);
> @@ -7618,6 +7650,11 @@ static int pqi_ctrl_init(struct pqi_ctrl_info *ctrl_info)
>   	u32 product_id;
>   
>   	if (reset_devices) {
> +		if (pqi_is_fw_triage_supported(ctrl_info)) {
> +			rc = sis_wait_for_fw_triage_completion(ctrl_info);
> +			if (rc)
> +				return rc;
> +		}
>   		sis_soft_reset(ctrl_info);
>   		msleep(PQI_POST_RESET_DELAY_SECS * PQI_HZ);
>   	} else {
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.c b/drivers/scsi/smartpqi/smartpqi_sis.c
> index d63c46a8e38b..8acd3a80f582 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.c
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.c
> @@ -51,12 +51,20 @@
>   #define SIS_BASE_STRUCT_REVISION		9
>   #define SIS_BASE_STRUCT_ALIGNMENT		16
>   
> +#define SIS_CTRL_KERNEL_FW_TRIAGE		0x3
>   #define SIS_CTRL_KERNEL_UP			0x80
>   #define SIS_CTRL_KERNEL_PANIC			0x100
>   #define SIS_CTRL_READY_TIMEOUT_SECS		180
>   #define SIS_CTRL_READY_RESUME_TIMEOUT_SECS	90
>   #define SIS_CTRL_READY_POLL_INTERVAL_MSECS	10
>   
> +enum sis_fw_triage_status {
> +	FW_TRIAGE_NOT_STARTED = 0,
> +	FW_TRIAGE_STARTED,
> +	FW_TRIAGE_COND_INVALID,
> +	FW_TRIAGE_COMPLETED
> +};
> +
>   #pragma pack(1)
>   
>   /* for use with SIS_CMD_INIT_BASE_STRUCT_ADDRESS command */
> @@ -419,12 +427,55 @@ u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info)
>   	return readl(&ctrl_info->registers->sis_driver_scratch);
>   }
>   
> +static inline enum sis_fw_triage_status
> +	sis_read_firmware_triage_status(struct pqi_ctrl_info *ctrl_info)
> +{
> +	return ((enum sis_fw_triage_status)(readl(&ctrl_info->registers->sis_firmware_status) &
> +		SIS_CTRL_KERNEL_FW_TRIAGE));
> +}
> +
>   void sis_soft_reset(struct pqi_ctrl_info *ctrl_info)
>   {
>   	writel(SIS_SOFT_RESET,
>   		&ctrl_info->registers->sis_host_to_ctrl_doorbell);
>   }
>   
> +#define SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS		300
> +#define SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS		1
> +
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info)
> +{
> +	int rc;
> +	enum sis_fw_triage_status status;
> +	unsigned long timeout;
> +
> +	timeout = (SIS_FW_TRIAGE_STATUS_TIMEOUT_SECS * PQI_HZ) + jiffies;
> +	while (1) {
> +		status = sis_read_firmware_triage_status(ctrl_info);
> +		if (status == FW_TRIAGE_COND_INVALID) {
> +			dev_err(&ctrl_info->pci_dev->dev,
> +				"firmware triage condition invalid\n");
> +			rc = -EINVAL;
> +			break;
> +		} else if (status == FW_TRIAGE_NOT_STARTED ||
> +			status == FW_TRIAGE_COMPLETED) {
> +			rc = 0;
> +			break;
> +		}
> +
> +		if (time_after(jiffies, timeout)) {
> +			dev_err(&ctrl_info->pci_dev->dev,
> +				"timed out waiting for firmware triage status\n");
> +			rc = -ETIMEDOUT;
> +			break;
> +		}
> +
> +		ssleep(SIS_FW_TRIAGE_STATUS_POLL_INTERVAL_SECS);
> +	}
> +
> +	return rc;
> +}
> +
>   static void __attribute__((unused)) verify_structures(void)
>   {
>   	BUILD_BUG_ON(offsetof(struct sis_base_struct,
> diff --git a/drivers/scsi/smartpqi/smartpqi_sis.h b/drivers/scsi/smartpqi/smartpqi_sis.h
> index d29c1352a826..c1db93054c86 100644
> --- a/drivers/scsi/smartpqi/smartpqi_sis.h
> +++ b/drivers/scsi/smartpqi/smartpqi_sis.h
> @@ -28,5 +28,6 @@ void sis_write_driver_scratch(struct pqi_ctrl_info *ctrl_info, u32 value);
>   u32 sis_read_driver_scratch(struct pqi_ctrl_info *ctrl_info);
>   void sis_soft_reset(struct pqi_ctrl_info *ctrl_info);
>   u32 sis_get_product_id(struct pqi_ctrl_info *ctrl_info);
> +int sis_wait_for_fw_triage_completion(struct pqi_ctrl_info *ctrl_info);
>   
>   #endif	/* _SMARTPQI_SIS_H */
>