mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Hannes Reinecke <hare@suse.de>
To: Mohamed Khalfella <mkhalfella@purestorage.com>,
	Keith Busch <kbusch@kernel.org>, Jens Axboe <axboe@kernel.dk>,
	Christoph Hellwig <hch@lst.de>, Sagi Grimberg <sagi@grimberg.me>
Cc: Naresh Gottumukkala <nareshgottumukkala83@gmail.com>,
	Paul Ely <paul.ely@broadcom.com>,
	Hannes Reinecke <hare@kernel.org>,
	Chaitanya Kulkarni <kch@nvidia.com>,
	James Smart <jsmart833426@gmail.com>,
	Randy Jennings <randyj@purestorage.com>,
	Daniel Wagner <dwagner@suse.de>,
	linux-nvme@lists.infradead.org, linux-kernel@vger.kernel.org
Subject: Re: [PATCH v7 13/17] nvme-fc: perform error recovery directly from ioerr_work
Date: Mon, 16 Nov 2026 08:37:27 +0100	[thread overview]
Message-ID: <0cec2aa3-62c2-411f-a51b-9a3f3d25820b@suse.de> (raw)
In-Reply-To: <20261114030454.2836543-14-mkhalfella@purestorage.com>

On 11/14/26 4:04 AM, Mohamed Khalfella wrote:
> Now that nvme_fc_start_ioerr_recovery() moves the controller to
> RESETTING before queueing ioerr_work, the nvme_reset_ctrl() call in
> nvme_fc_error_recovery() fails with -EBUSY and no recovery runs.
> 
> Do the recovery in the work itself instead: stop the keep-alive and
> the AEN submission, tear down the association so that any command a
> core work item is waiting on is aborted at once, stop the controller,
> move to CONNECTING and schedule a reconnect. If the CONNECTING
> transition fails the controller is being deleted and the delete path
> finishes the job. This is how rdma and tcp structure their error
> recovery too.
> 
> The work is only ever queued right after a successful transition to
> RESETTING, so when it runs it owns the controller: a reset or a
> firmware activation holding RESETTING cannot be in progress at the
> same time, and the work does not need to re-check the state.
> 
> Signed-off-by: Mohamed Khalfella <mkhalfella@purestorage.com>
> ---
>   drivers/nvme/host/fc.c | 57 ++++++++++++++++--------------------------
>   1 file changed, 21 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
> index c4603be38144..0ba621649afe 100644
> --- a/drivers/nvme/host/fc.c
> +++ b/drivers/nvme/host/fc.c
> @@ -229,6 +229,8 @@ static struct device *fc_udev_device;
>   static void nvme_fc_complete_rq(struct request *rq);
>   static void nvme_fc_start_ioerr_recovery(struct nvme_fc_ctrl *ctrl,
>   					 char *errmsg);
> +static void __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl,
> +					    bool start_queues);
>   
>   /* *********************** FC-NVME Port Management ************************ */
>   
> @@ -987,7 +989,8 @@ fc_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents,
>   static void nvme_fc_ctrl_put(struct nvme_fc_ctrl *);
>   static int nvme_fc_ctrl_get(struct nvme_fc_ctrl *);
>   
> -static void nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg);
> +static void nvme_fc_delete_association(struct nvme_fc_ctrl *ctrl);
> +static void nvme_fc_reconnect_or_delete(struct nvme_fc_ctrl *ctrl, int status);
>   
>   static void
>   __nvme_fc_finish_ls_req(struct nvmefc_ls_req_op *lsop)
> @@ -1880,7 +1883,23 @@ nvme_fc_ctrl_ioerr_work(struct work_struct *work)
>   	struct nvme_fc_ctrl *ctrl =
>   			container_of(work, struct nvme_fc_ctrl, ioerr_work);
>   
> -	nvme_fc_error_recovery(ctrl, "transport detected io error");

Hmm. _Some_ message would be nice here.

> +	nvme_stop_keep_alive(&ctrl->ctrl);
> +	flush_work(&ctrl->ctrl.async_event_work);
> +
> +	/* will block while waiting for io to terminate */
> +	nvme_fc_delete_association(ctrl);
> +	nvme_stop_ctrl(&ctrl->ctrl);
> +
> +	/* Do not reconnect if controller is being deleted */
> +	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING))
> +		return;
Same here; it's the same condition than the original 
nvme_fc_error_recovery() checked for, so we should keep the
logging message.

> +
> +	if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE) {
> +		queue_delayed_work(nvme_wq, &ctrl->connect_work, 0);
> +		return;
> +	}
> +
> +	nvme_fc_reconnect_or_delete(ctrl, -ENOTCONN);

And you can move this under the 'else' condition from the above
if-clause.

>   }
>   
>   /*
> @@ -2528,39 +2547,6 @@ __nvme_fc_abort_outstanding_ios(struct nvme_fc_ctrl *ctrl, bool start_queues)
>   		nvme_unquiesce_admin_queue(&ctrl->ctrl);
>   }
>   
> -static void
> -nvme_fc_error_recovery(struct nvme_fc_ctrl *ctrl, char *errmsg)
> -{
> -	enum nvme_ctrl_state state = nvme_ctrl_state(&ctrl->ctrl);
> -
> -	/*
> -	 * if an error (io timeout, etc) while (re)connecting, the remote
> -	 * port requested terminating of the association (disconnect_ls)
> -	 * or an error (timeout or abort) occurred on an io while creating
> -	 * the controller.  Abort any ios on the association and let the
> -	 * create_association error path resolve things.
> -	 */
> -	if (state == NVME_CTRL_CONNECTING) {
> -		__nvme_fc_abort_outstanding_ios(ctrl, true);
> -		dev_warn(ctrl->ctrl.device,
> -			"NVME-FC{%d}: transport error during (re)connect\n",
> -			ctrl->cnum);
> -		return;
> -	}
> -
> -	/* Otherwise, only proceed if in LIVE state - e.g. on first error */
> -	if (state != NVME_CTRL_LIVE)
> -		return;
> -
> -	dev_warn(ctrl->ctrl.device,
> -		"NVME-FC{%d}: transport association event: %s\n",
> -		ctrl->cnum, errmsg);
> -	dev_warn(ctrl->ctrl.device,
> -		"NVME-FC{%d}: resetting controller\n", ctrl->cnum);
> -
> -	nvme_reset_ctrl(&ctrl->ctrl);
> -}
> -
>   static enum blk_eh_timer_return nvme_fc_timeout(struct request *rq)
>   {
>   	struct nvme_fc_fcp_op *op = blk_mq_rq_to_pdu(rq);
> @@ -3399,7 +3385,6 @@ nvme_fc_reset_ctrl_work(struct work_struct *work)
>   	}
>   }
>   
> -
>   static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = {
>   	.name			= "fc",
>   	.module			= THIS_MODULE,

Cheers,

Hannes
-- 
Dr. Hannes Reinecke                  Kernel Storage Architect
hare@suse.de                                +49 911 74053 688
SUSE Software Solutions GmbH, Frankenstr. 146, 90461 Nürnberg
HRB 36809 (AG Nürnberg), GF: I. Totev, A. McDonald, W. Knoblich

  reply	other threads:[~2026-11-16  7:37 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-14  3:04 [PATCH v7 00/17] TP8028 Rapid Path Failure Recovery Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 01/17] nvmet: Rapid Path Failure Recovery set controller identify fields Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 02/17] nvmet/debugfs: Export controller CIU and CIRN via debugfs Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 03/17] nvmet: Implement CCR nvme command Mohamed Khalfella
2026-11-16  8:45   ` Dmitry Bogdanov
2026-11-14  3:04 ` [PATCH v7 04/17] nvmet: Implement CCR logpage Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 05/17] nvmet: Send an AEN on CCR completion Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 06/17] nvme: Rapid Path Failure Recovery read controller identify fields Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 07/17] nvme: Introduce FENCING and FENCED controller states Mohamed Khalfella
2026-11-16  7:16   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 08/17] nvme: Implement cross-controller reset recovery Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 09/17] nvme: Implement cross-controller reset completion Mohamed Khalfella
2026-11-16  7:17   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 10/17] nvme-tcp: Use CCR to recover controller that hits an error Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 11/17] nvme-rdma: " Mohamed Khalfella
2026-11-16  7:19   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 12/17] nvme-fc: start error recovery instead of aborting timed out IOs Mohamed Khalfella
2026-11-16  7:31   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 13/17] nvme-fc: perform error recovery directly from ioerr_work Mohamed Khalfella
2026-11-16  7:37   ` Hannes Reinecke [this message]
2026-11-14  3:04 ` [PATCH v7 14/17] nvme-fc: Use CCR to recover controller that hits an error Mohamed Khalfella
2026-11-16  7:38   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 15/17] nvme-fc: Hold inflight requests while in FENCING state Mohamed Khalfella
2026-11-16  7:52   ` Hannes Reinecke
2026-11-14  3:04 ` [PATCH v7 16/17] nvmet: Add support for CQT to nvme target Mohamed Khalfella
2026-11-14  3:04 ` [PATCH v7 17/17] nvme: Add support for CQT to nvme host Mohamed Khalfella
2026-11-16  7:55   ` Hannes Reinecke

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=0cec2aa3-62c2-411f-a51b-9a3f3d25820b@suse.de \
    --to=hare@suse.de \
    --cc=axboe@kernel.dk \
    --cc=dwagner@suse.de \
    --cc=hare@kernel.org \
    --cc=hch@lst.de \
    --cc=jsmart833426@gmail.com \
    --cc=kbusch@kernel.org \
    --cc=kch@nvidia.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-nvme@lists.infradead.org \
    --cc=mkhalfella@purestorage.com \
    --cc=nareshgottumukkala83@gmail.com \
    --cc=paul.ely@broadcom.com \
    --cc=randyj@purestorage.com \
    --cc=sagi@grimberg.me \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®