mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Richard Cheng <icheng@nvidia.com>
To: Srirangan Madhavan <smadhavan@nvidia.com>
Cc: Alison Schofield <alison.schofield@intel.com>,
	 Bjorn Helgaas <bhelgaas@google.com>,
	Dan Williams <djbw@kernel.org>,
	 Dave Jiang <dave.jiang@intel.com>,
	Davidlohr Bueso <dave@stgolabs.net>,
	 Ira Weiny <ira.weiny@intel.com>,
	Jonathan Cameron <jic23@kernel.org>,
	 Vishal Verma <vishal.l.verma@intel.com>,
	linux-cxl@vger.kernel.org, linux-pci@vger.kernel.org,
	 linux-kernel@vger.kernel.org,
	Alex Williamson <alex.williamson@redhat.com>,
	vsethi@nvidia.com,  alwilliamson@nvidia.com,
	Dan Williams <danwilliams@nvidia.com>,
	 Sai Yashwanth Reddy Kancherla <skancherla@nvidia.com>,
	Vishal Aslot <vaslot@nvidia.com>,
	 Manish Honap <mhonap@nvidia.com>, Jiandi An <jan@nvidia.com>,
	linux-tegra@vger.kernel.org
Subject: Re: [PATCH v8 07/10] PCI/CXL: Discover the CXL reset scope
Date: Tue, 7 Jul 2026 09:45:33 +0800	[thread overview]
Message-ID: <akxYmEuXX7W1v5DS@MWDK4CY14F> (raw)
In-Reply-To: <20260703220508.546528-8-smadhavan@nvidia.com>

On Fri, Jul 03, 2026 at 10:05:05PM +0800, Srirangan Madhavan wrote:
> Add reset context support to validate that CXL Reset is function-scoped
> before advertising it as a PCI reset method. Use the Non-CXL Function
> Map, ARI/devfn rules, and CXL.cache/mem capability bits to reject reset
> when another same-scope function would also be affected.
> 
> If the Function Map cannot be read, warn and conservatively treat all
> candidate same-scope functions as CXL-capable for scope validation.
> 
> Signed-off-by: Srirangan Madhavan <smadhavan@nvidia.com>
> ---
>  drivers/cxl/core/reset.c      | 153 +++++++++++++++++++++++++++++++++-
>  include/uapi/linux/pci_regs.h |   1 +
>  2 files changed, 152 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/cxl/core/reset.c b/drivers/cxl/core/reset.c
> index fb741b66e502..8e80e7588046 100644
> --- a/drivers/cxl/core/reset.c
> +++ b/drivers/cxl/core/reset.c
> @@ -1,5 +1,6 @@
>  // SPDX-License-Identifier: GPL-2.0-only
>  /* Copyright(c) 2026 NVIDIA Corporation. All rights reserved. */
> +#include <linux/bitmap.h>
>  #include <linux/delay.h>
>  #include <linux/bug.h>
>  #include <linux/bitfield.h>
> @@ -469,6 +470,21 @@ static const u32 cxl_reset_timeout_ms[] = {
>  #define CXL_CACHE_WBI_TIMEOUT_US 100000
>  #define CXL_CACHE_WBI_POLL_US 100
>  
> +/* CXL r4.0 sec 8.1.4 defines 256 bits of Non-CXL Function Map. */
> +#define CXL_RESET_MAX_FUNCTIONS 256
> +#define CXL_RESET_FUNCTION_MAP_REGS (CXL_RESET_MAX_FUNCTIONS / 32)
> +
> +struct cxl_reset_context {
> +	struct pci_dev *target;
> +};
> +
> +struct cxl_reset_walk_context {
> +	struct cxl_reset_context *ctx;
> +	DECLARE_BITMAP(non_cxl_func_map, CXL_RESET_MAX_FUNCTIONS);
> +	bool ari;
> +	int rc;
> +};
> +
>  struct cxl_hdm_range {
>  	struct list_head list;
>  	struct pci_dev *pdev;
> @@ -480,6 +496,137 @@ struct cxl_hdm_range_context {
>  	struct list_head ranges;
>  };
>  
> +static void cxl_reset_context_init(struct cxl_reset_context *ctx,
> +				   struct pci_dev *pdev)
> +{
> +	*ctx = (struct cxl_reset_context) {
> +		.target = pdev,
> +	};
> +}
> +
> +static void cxl_reset_read_non_cxl_func_map(struct pci_dev *pdev,
> +					    unsigned long *map)
> +{
> +	u32 words[CXL_RESET_FUNCTION_MAP_REGS];
> +	int dvsec, reg;
> +
> +	bitmap_zero(map, CXL_RESET_MAX_FUNCTIONS);
> +
> +	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> +					  PCI_DVSEC_CXL_FUNCTION_MAP);
> +	if (!dvsec)
> +		return;
> +
> +	for (reg = 0; reg < CXL_RESET_FUNCTION_MAP_REGS; reg++) {
> +		int rc;
> +
> +		rc = pci_read_config_dword(pdev,
> +					   dvsec + PCI_DVSEC_CXL_FUNCTION_MAP_REG +
> +					   reg * sizeof(u32), &words[reg]);
> +		if (rc) {
> +			pci_warn(pdev,
> +				 "failed to read Non-CXL Function Map; treating same-scope functions as CXL\n");
> +			bitmap_zero(map, CXL_RESET_MAX_FUNCTIONS);
> +			return;
> +		}
> +	}
> +
> +	bitmap_from_arr32(map, words, CXL_RESET_MAX_FUNCTIONS);
> +}
> +
> +static int cxl_reset_func_map_bit(struct pci_dev *sibling, bool ari)
> +{
> +	if (ari)
> +		return sibling->devfn;
> +
> +	/*
> +	 * Without ARI, the Function Map is organized as 32 device slots per
> +	 * conventional 3-bit function number.
> +	 */
> +	return PCI_FUNC(sibling->devfn) * 32 + PCI_SLOT(sibling->devfn);
> +}
> +
> +static int cxl_reset_read_cxl_cap(struct pci_dev *pdev, u16 *cap)
> +{
> +	int dvsec, rc;
> +
> +	dvsec = pci_find_dvsec_capability(pdev, PCI_VENDOR_ID_CXL,
> +					  PCI_DVSEC_CXL_DEVICE);
> +	if (!dvsec)
> +		return -ENODEV;
> +
> +	rc = pci_read_config_word(pdev, dvsec + PCI_DVSEC_CXL_CAP, cap);
> +	if (rc) {
> +		rc = pcibios_err_to_errno(rc);
> +		pci_warn(pdev, "failed to read CXL capability: %d\n", rc);
> +		return rc;
> +	}
> +
> +	return 0;
> +}
> +
> +static int cxl_reset_has_cache_or_mem(struct pci_dev *pdev)
> +{
> +	u16 cap;
> +	int rc;
> +
> +	rc = cxl_reset_read_cxl_cap(pdev, &cap);
> +	if (rc == -ENODEV)
> +		return 0;
> +	if (rc)
> +		return rc;
> +
> +	return !!(cap & (PCI_DVSEC_CXL_CACHE_CAPABLE |
> +			 PCI_DVSEC_CXL_MEM_CAPABLE));
> +}
> +
> +static int cxl_reset_validate_function_scope(struct pci_dev *sibling,
> +					     void *data)
> +{
> +	struct cxl_reset_walk_context *wctx = data;
> +	struct cxl_reset_context *ctx = wctx->ctx;
> +	struct pci_dev *pdev = ctx->target;
> +	int fn, rc;
> +
> +	if (sibling == pdev)
> +		return 0;
> +
> +	if (sibling->bus != pdev->bus)
> +		return 0;
> +
> +	if (!wctx->ari && PCI_SLOT(sibling->devfn) != PCI_SLOT(pdev->devfn))
> +		return 0;
> +
> +	fn = cxl_reset_func_map_bit(sibling, wctx->ari);
> +	if (test_bit(fn, wctx->non_cxl_func_map))
> +		return 0;
> +
> +	rc = cxl_reset_has_cache_or_mem(sibling);
> +	if (rc < 0) {
> +		wctx->rc = rc;
> +		return rc;
> +	}
> +	if (!rc)
> +		return 0;
> +

cxl_reset_has_cache_or_mem() treats a sibling with no CXL device DVSEC as non-CXL, but according to
spec, it places the DVSEC only on Function 0, the sibling function can participate under fn0's DVSEC
with no DVSEC of their own. The non-CXL Function Map should be authoritative for classifying siblings
rather than a per-sibling DVSEC read.

What do you think ?

--Richard

> +	wctx->rc = -ENOTTY;
> +	return wctx->rc;
> +}
> +
> +static int cxl_reset_validate_function_scoped(struct cxl_reset_context *ctx)
> +{
> +	struct pci_dev *pdev = ctx->target;
> +	struct cxl_reset_walk_context wctx = {
> +		.ctx = ctx,
> +		.ari = pci_ari_enabled(pdev->bus),
> +	};
> +
> +	cxl_reset_read_non_cxl_func_map(pdev, wctx.non_cxl_func_map);
> +	pci_walk_bus(pdev->bus, cxl_reset_validate_function_scope, &wctx);
> +
> +	return wctx.rc;
> +}
> +
>  static void cxl_hdm_range_context_init(struct cxl_hdm_range_context *ctx)
>  {
>  	INIT_LIST_HEAD(&ctx->ranges);
> @@ -890,6 +1037,7 @@ static int cxl_reset_execute(struct pci_dev *pdev, int dvsec)
>  int cxl_reset_function(struct pci_dev *pdev, bool probe)
>  {
>  	struct cxl_hdm_range_context range_ctx;
> +	struct cxl_reset_context ctx;
>  	int dvsec;
>  	int rc;
>  
> @@ -897,8 +1045,9 @@ int cxl_reset_function(struct pci_dev *pdev, bool probe)
>  	if (dvsec < 0)
>  		return dvsec;
>  
> +	cxl_reset_context_init(&ctx, pdev);
>  	if (probe)
> -		return 0;
> +		return cxl_reset_validate_function_scoped(&ctx);
> 

This only runs on the PCI_RESET_PROBE path, but pci_init_reset_methods() runs during pci_device_add(),
before sibling functions of a multi-function device are on pdev->bus.
Thus pci_walk_bus() at probe time walks on an incomplete sibling set and passes, the DO_RESET path never
re-validates.
A multi-function Type-2 device would advertise and perform cxl_reset even with a live CXL sibling.
Can the scope be recheck at reset time or gate on siblings actually being present ?

 
>  	cxl_hdm_range_context_init(&range_ctx);
>  
> @@ -907,7 +1056,7 @@ int cxl_reset_function(struct pci_dev *pdev, bool probe)
>  		if (!rc)
>  			rc = cxl_reset_execute(pdev, dvsec);
>  	}
> -
> +out:
>  	cxl_hdm_range_context_destroy(&range_ctx);
>  	return rc;
>  }
> diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
> index 194ae56b4404..7fc1d34fcce7 100644
> --- a/include/uapi/linux/pci_regs.h
> +++ b/include/uapi/linux/pci_regs.h
> @@ -1380,6 +1380,7 @@
>  
>  /* CXL r4.0, 8.1.4: Non-CXL Function Map DVSEC */
>  #define PCI_DVSEC_CXL_FUNCTION_MAP			2
> +#define  PCI_DVSEC_CXL_FUNCTION_MAP_REG			0x0C
>  
>  /* CXL r4.0, 8.1.5: Extensions DVSEC for Ports */
>  #define PCI_DVSEC_CXL_PORT				3
> -- 
> 2.43.0
> 

  reply	other threads:[~2026-07-07  1:45 UTC|newest]

Thread overview: 27+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-07-03 22:04 [PATCH v8 00/10] PCI/CXL: Add CXL reset support for Type 2 devices Srirangan Madhavan
2026-07-03 22:04 ` [PATCH v8 01/10] cxl: Split decoder programming into a reusable helper Srirangan Madhavan
2026-07-07  0:22   ` Dave Jiang
2026-07-09  0:48     ` Srirangan Madhavan
2026-07-03 22:05 ` [PATCH v8 02/10] cxl: Cache decoder settings on PCI devices Srirangan Madhavan
2026-07-07 16:45   ` Dave Jiang
2026-07-03 22:05 ` [PATCH v8 03/10] cxl: Share HDM decoder decode logic Srirangan Madhavan
2026-07-03 22:05 ` [PATCH v8 04/10] cxl: Cache endpoint decoder settings during PCI enumeration Srirangan Madhavan
2026-07-07  5:30   ` Richard Cheng
2026-07-07 17:47   ` Dave Jiang
2026-07-03 22:05 ` [PATCH v8 05/10] cxl: Add CXL Device Reset helper Srirangan Madhavan
2026-07-07 19:14   ` Dave Jiang
2026-07-03 22:05 ` [PATCH v8 06/10] cxl: Validate HDM ranges before CXL reset Srirangan Madhavan
2026-07-07 21:34   ` Dave Jiang
2026-07-03 22:05 ` [PATCH v8 07/10] PCI/CXL: Discover the CXL reset scope Srirangan Madhavan
2026-07-07  1:45   ` Richard Cheng [this message]
2026-07-03 22:05 ` [PATCH v8 08/10] cxl: Restore CXL HDM state after PCI reset Srirangan Madhavan
2026-07-07  1:34   ` Richard Cheng
2026-07-07 22:30   ` Dave Jiang
2026-07-03 22:05 ` [PATCH v8 09/10] PCI/CXL: Expose CXL Reset as a PCI reset method Srirangan Madhavan
2026-07-03 22:05 ` [PATCH v8 10/10] Documentation/ABI: Document CXL Reset " Srirangan Madhavan
2026-07-03 22:08 ` [PATCH RFC] PCI/CXL: Restore HDM state after CXL bus reset Srirangan Madhavan
2026-07-06 23:13 ` [PATCH v8 00/10] PCI/CXL: Add CXL reset support for Type 2 devices Dave Jiang
2026-07-07 22:45   ` Dave Jiang
2026-07-09  0:34     ` Srirangan Madhavan
2026-07-09 15:32       ` Dave Jiang
2026-07-07  5:41 ` Richard Cheng

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=akxYmEuXX7W1v5DS@MWDK4CY14F \
    --to=icheng@nvidia.com \
    --cc=alex.williamson@redhat.com \
    --cc=alison.schofield@intel.com \
    --cc=alwilliamson@nvidia.com \
    --cc=bhelgaas@google.com \
    --cc=danwilliams@nvidia.com \
    --cc=dave.jiang@intel.com \
    --cc=dave@stgolabs.net \
    --cc=djbw@kernel.org \
    --cc=ira.weiny@intel.com \
    --cc=jan@nvidia.com \
    --cc=jic23@kernel.org \
    --cc=linux-cxl@vger.kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=linux-tegra@vger.kernel.org \
    --cc=mhonap@nvidia.com \
    --cc=skancherla@nvidia.com \
    --cc=smadhavan@nvidia.com \
    --cc=vaslot@nvidia.com \
    --cc=vishal.l.verma@intel.com \
    --cc=vsethi@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®