Re: [RFC PATCH v2 3/5] PCI/TSM: Derive the coherent-range IPA from CXL

[Richard Cheng <icheng@nvidia.com>]

On Thu, Nov 05, 2026 at 03:29:07PM +0800, Ankit Agrawal wrote:
> The DevIf report range with range ID 0xffff describes the device's
> coherent (CXL) memory window and unlike every other reported range
> carries no BAR number. Recognize
> PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_COHERENT and recover that window's
> address from the CXL side of the device instead of treating it as a
> BAR-relative offset.
> 
> pci_tsm_retrieve_coherent_range() resolves the window from
> pdev->coh_resource[], snapshotted once with the committed CXL HDM
> decoders range. The range to IPA conversion is applied onto
> range_base/range_len using pdev->coh_resource[]. The ascending BAR order
> checks stay on the BAR path only. The coherent range is required to be
> the last report entry and is not part of the BAR sequence.
> 
> An HDM decoded window lives in the CXL Fixed Memory Window and is not
> part of a BAR. So by construction, no pci_dev resource contains it. So
> the function pci_tsm_coh_resource_contains() validates the range against
> pdev->coh_resource[] instead; the same known-good snapshot
> pci_tsm_retrieve_coherent_range() already trusts.
> 
> Signed-off-by: Ankit Agrawal <ankita@nvidia.com>
> ---
>  drivers/pci/Kconfig     |  12 ++++
>  drivers/pci/tsm.c       | 119 ++++++++++++++++++++++++++++------------
>  include/linux/pci-tsm.h |  75 +++++++++++++++++++++++++
>  3 files changed, 170 insertions(+), 36 deletions(-)
> 
> diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
> index 16fbd4212e0f..22c416c84a6f 100644
> --- a/drivers/pci/Kconfig
> +++ b/drivers/pci/Kconfig
> @@ -139,6 +139,18 @@ config PCI_TSM
>  
>  	  Enable a platform TSM driver to use this capability.
>  
> +config PCI_TSM_CXL_COHERENT
> +	bool "PCI TSM: CXL coherent range support"
> +	depends on PCI_TSM && CXL_RESET
> +	default y
> +	help
> +	  Enables helper functions to manage coherent regions on CXL devices
> +	  for TSM. The device CXL coherent (CXL.mem) ranges could be tracked
> +	  and fetched across subsystems to be reported in a TDISP Device
> +	  Interface Report.
> +
> +	  If unsure, say Y.
> +
>  config PCI_DOE
>  	bool "Enable PCI Data Object Exchange (DOE) support"
>  	help
> diff --git a/drivers/pci/tsm.c b/drivers/pci/tsm.c
> index d074bfd18d40..bc3218c58ff0 100644
> --- a/drivers/pci/tsm.c
> +++ b/drivers/pci/tsm.c
> @@ -939,8 +939,9 @@ int pci_tsm_mmio_setup(struct pci_dev *pdev, struct pci_tsm_mmio *mmio)
>  		return -EBUSY;
>  
>  	for (i = 0; i < mmio->nr; i++) {
> -		struct resource *res = pci_tsm_mmio_resource(mmio, i);
> -		int j;
> +		struct pci_tsm_mmio_entry *entry = pci_tsm_mmio_entry(mmio, i);
> +		struct resource *res = &entry->res;
> +		bool coherent = entry->flags & PCI_TSM_MMIO_F_COHERENT;
>  
>  		if (resource_size(res) == 0 || !(res->flags & IORESOURCE_MEM))
>  			break;
> @@ -950,13 +951,28 @@ int pci_tsm_mmio_setup(struct pci_dev *pdev, struct pci_tsm_mmio *mmio)
>  					     pci_name(pdev), IORESOURCE_MEM,
>  					     IORES_DESC_ENCRYPTED);
>  
> -		for (j = 0; j < PCI_NUM_RESOURCES; j++)
> -			if (resource_contains(pci_resource_n(pdev, j), res))
> +		/*
> +		 * The coherent (CXL.mem) range isn't BAR-backed, so check it
> +		 * against pdev->coh_resource[] instead of pci_resource_n().
> +		 */
> +		if (coherent) {
> +			if (!pci_tsm_coh_resource_contains(pdev, res))
>  				break;
> -
> -		/* Request is outside of device MMIO */
> -		if (j >= PCI_NUM_RESOURCES)
> -			break;
> +		} else {
> +			struct resource *r;
> +			bool contained = false;
> +
> +			pci_dev_for_each_resource(pdev, r) {
> +				if (resource_contains(r, res)) {
> +					contained = true;
> +					break;
> +				}
> +			}
> +
> +			/* Request is outside of device MMIO */
> +			if (!contained)
> +				break;
> +		}
>  
>  		if (insert_resource(&encrypted_iomem_resource, res) != 0)
>  			break;
> @@ -988,6 +1004,7 @@ EXPORT_SYMBOL_GPL(pci_tsm_mmio_teardown);
>  #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_IS_NON_TEE BIT(2)
>  #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_IS_UPDATABLE BIT(3)
>  #define PCI_TSM_DEVIF_REPORT_MMIO_ATTR_RANGE_ID GENMASK(31, 16)
> +#define PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_COHERENT 0xffff
>  
>  /* An interface report 'pfn' is 4K in size */
>  struct pci_tsm_devif_mmio {
> @@ -1043,6 +1060,7 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
>  		return NULL;
>  
>  	for (i = 0; i < mmio_range_count; i++) {
> +		struct range window;
>  		u64 range_off;
>  		struct range range;
>  		const struct pci_tsm_devif_mmio *mmio_data = &devif_report->mmio[i];
> @@ -1053,33 +1071,58 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
>  		u32 attr = __le32_to_cpu(mmio_data->attributes);
>  		int bar = FIELD_GET(PCI_TSM_DEVIF_REPORT_MMIO_ATTR_RANGE_ID,
>  				    attr);
> +		bool coherent =
> +			bar == PCI_TSM_DEVIF_REPORT_MMIO_RANGE_ID_COHERENT;
>  
> -		if (bar >= PCI_STD_NUM_BARS ||
> -		    !(pci_resource_flags(pdev, bar) & IORESOURCE_MEM) ||
> -		    (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) {
> -			pci_dbg(pdev, "Invalid reporting bar ID %d\n", bar);
> -			return NULL;
> -		}
> -
> -		if (last_bar > bar) {
> -			pci_dbg(pdev, "Reporting bar ID not in ascending order\n");
> -			return NULL;
> -		}
> -
> -		if (last_bar < bar) {
> -			resource_size_t mask = pci_resource_len(pdev, bar) - 1;
> -
> -			/* Transition to a new bar */
> -			last_bar = bar;
> +		if (coherent) {
> +			if (i != mmio_range_count - 1) {
> +				pci_dbg(pdev, "Coherent reporting range is not last\n");
> +				return NULL;
> +			}
>  
>  			/*
> -			 * Determine the obfuscated base of the BAR. BAR
> -			 * offsets are never obfuscated.
> +			 * No BAR names the coherent range, so fail closed if
> +			 * neither a CXL region nor a committed HDM decoder
> +			 * resolves its address.
>  			 */
> -			reporting_bar_base = tsm_offset & ~mask;
> -		} else if (tsm_offset < last_reporting_end) {
> -			pci_dbg(pdev, "Reporting ranges within BAR not in ascending order\n");
> -			return NULL;
> +			if (pci_tsm_retrieve_coherent_range(pdev, &window)) {
> +				pci_dbg(pdev, "No CXL region or committed HDM decoder for coherent reporting range\n");
> +				return NULL;
> +			}
> +
> +			reporting_bar_base = tsm_offset &
> +					     ~((u64)range_len(&window) - 1);

Hi Ankit,

HDM decoder window can have non-power-of-2 sizes, why do we assume a power-of-2 size for
coherent window?

E.g. if we have something like the following
Window start: 4GB
Window size: 768 MB
Reported address: 4GB + 256 MB

Assume MMIO_REPORTING_OFFSET is 0, so no address obfuscation needs to be removed.
The reported page is simply 256 MB into the window.

For this size, the mask leaves the reported address unchanged.

0x110000000 & ~(0x30000000 - 1) = 0x110000000

reporting_bar_base becomes 4GB + 256 MB, rather than actual window base of 4 GB, so
range_off is 0.

The fianl address will be window.start + range_off = 4GB + 0

A KUnit test against your patch series can reproduce this.
Furthermore, pci_tsm_mmio_setup() accepts the wrong result, because the incorrect address
is still insdie the window.

Best regards,
Richard Cheng.

> +		} else {
> +			if (bar >= PCI_STD_NUM_BARS ||
> +			    !(pci_resource_flags(pdev, bar) & IORESOURCE_MEM) ||
> +			    (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET)) {
> +				pci_dbg(pdev, "Invalid reporting bar ID %d\n", bar);
> +				return NULL;
> +			}
> +
> +			if (last_bar > bar) {
> +				pci_dbg(pdev, "Reporting bar ID not in ascending order\n");
> +				return NULL;
> +			}
> +
> +			if (last_bar < bar) {
> +				resource_size_t mask = pci_resource_len(pdev, bar) - 1;
> +
> +				/* Transition to a new bar */
> +				last_bar = bar;
> +
> +				/*
> +				 * Determine the obfuscated base of the BAR. BAR
> +				 * offsets are never obfuscated.
> +				 */
> +				reporting_bar_base = tsm_offset & ~mask;
> +			} else if (tsm_offset < last_reporting_end) {
> +				pci_dbg(pdev, "Reporting ranges within BAR not in ascending order\n");
> +				return NULL;
> +			}
> +
> +			window.start = pci_resource_start(pdev, bar);
> +			window.end = pci_resource_end(pdev, bar);
>  		}
>  
>  		/* Per spec the tsm_offset never results in overflow / underflow */
> @@ -1090,19 +1133,21 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
>  		}
>  
>  		range_off = tsm_offset - reporting_bar_base;
> -		if (pci_resource_len(pdev, bar) < range_off + size) {
> -			pci_dbg(pdev, "Reporting range larger than BAR size\n");
> +
> +		if (range_len(&window) < range_off + size) {
> +			pci_dbg(pdev, "Reporting range larger than %s size\n",
> +				coherent ? "coherent range" : "BAR");
>  			return NULL;
>  		}
>  
> -		range.start = pci_resource_start(pdev, bar) + range_off;
> +		range.start = window.start + range_off;
>  		range.end = range.start + size - 1;
>  
>  		/* Only record the TEE ranges for later consideration by ioremap() */
>  		if (FIELD_GET(PCI_TSM_DEVIF_REPORT_MMIO_ATTR_IS_NON_TEE,
>  			      attr)) {
> -			pci_dbg(pdev, "Skipping non-TEE range, BAR%d %pra\n",
> -				bar, &range);
> +			pci_dbg(pdev, "Skipping non-TEE range, %s %pra\n",
> +				coherent ? "coherent range" : "BAR", &range);
>  			continue;
>  		}
>  
> @@ -1110,6 +1155,8 @@ struct pci_tsm_mmio *pci_tsm_mmio_alloc(struct pci_dev *pdev,
>  		entry->res.end = range.end;
>  		entry->res.flags = IORESOURCE_MEM;
>  		entry->tsm_offset = tsm_offset;
> +		if (coherent)
> +			entry->flags |= PCI_TSM_MMIO_F_COHERENT;
>  		mmio->nr++;
>  	}
>  
> diff --git a/include/linux/pci-tsm.h b/include/linux/pci-tsm.h
> index e54ad057fa8e..7c29f10e7795 100644
> --- a/include/linux/pci-tsm.h
> +++ b/include/linux/pci-tsm.h
> @@ -1,8 +1,10 @@
>  /* SPDX-License-Identifier: GPL-2.0 */
>  #ifndef __PCI_TSM_H
>  #define __PCI_TSM_H
> +#include <linux/bits.h>
>  #include <linux/mutex.h>
>  #include <linux/pci.h>
> +#include <linux/range.h>
>  #include <linux/sockptr.h>
>  
>  struct pci_tsm;
> @@ -199,15 +201,20 @@ enum pci_tsm_req_scope {
>  	PCI_TSM_REQ_DEBUG_WRITE = 3,
>  };
>  
> +/* The range is the device's CXL coherent (CXL.mem) window, not BAR-backed */
> +#define PCI_TSM_MMIO_F_COHERENT BIT(0)
> +
>  /**
>   * struct pci_tsm_mmio_entry - an encrypted MMIO range
>   * @res: MMIO address range (typically Guest Physical Address, GPA)
>   * @tsm_offset: Host Physical Address, HPA obfuscation offset added by the TSM.
>   *		Translates report addresses to GPA.
> + * @flags: PCI_TSM_MMIO_F_* attributes for the range
>   */
>  struct pci_tsm_mmio_entry {
>  	struct resource res;
>  	u64 tsm_offset;
> +	u32 flags;
>  };
>  
>  struct pci_tsm_mmio {
> @@ -227,6 +234,74 @@ static inline struct resource *pci_tsm_mmio_resource(struct pci_tsm_mmio *mmio,
>  	return &mmio->mmio[idx].res;
>  }
>  
> +#ifdef CONFIG_PCI_TSM_CXL_COHERENT
> +/*
> + * pci_tsm_coh_resource_contains() - check a coherent range against the
> + * PCI-core coh_resource[] snapshot
> + * @pdev: device owner of @res
> + * @res: candidate coherent MMIO range to validate
> + *
> + * The coherent (CXL.mem) range isn't BAR-backed. So pci_resource_n()
> + * containment doesn't apply. Check against pdev->coh_resource[] instead.
> + *
> + * Return: true if @res falls within a populated coh_resource[] entry.
> + */
> +static inline bool pci_tsm_coh_resource_contains(struct pci_dev *pdev,
> +						 const struct resource *res)
> +{
> +	for (int i = 0; i < PCI_CXL_MAX_COHERENT_RANGES; i++) {
> +		struct resource *coh_res = &pdev->coh_resource[i];
> +
> +		if (resource_type(coh_res) == IORESOURCE_MEM &&
> +		    resource_contains(coh_res, res))
> +			return true;
> +	}
> +
> +	return false;
> +}
> +
> +/*
> + * pci_tsm_retrieve_coherent_range() - resolve the device's coherent CXL window
> + * @pdev: device owner of the reported ranges
> + * @out: host physical (guest IPA) range of the coherent window
> + *
> + * The coherent range carries no BAR number, so its address comes from
> + * pdev->coh_resource[], a pre-driver-bind snapshot of committed CXL HDM
> + * decoders.
> + *
> + * Return: 0 with *@out set from the first populated entry, or -ENODEV if no
> + * entry is populated.
> + */
> +static inline int pci_tsm_retrieve_coherent_range(struct pci_dev *pdev,
> +						  struct range *out)
> +{
> +	for (int i = 0; i < PCI_CXL_MAX_COHERENT_RANGES; i++) {
> +		struct resource *res = &pdev->coh_resource[i];
> +
> +		if (resource_type(res) != IORESOURCE_MEM)
> +			continue;
> +
> +		out->start = res->start;
> +		out->end = res->end;
> +		return 0;
> +	}
> +
> +	return -ENODEV;
> +}
> +#else
> +static inline bool pci_tsm_coh_resource_contains(struct pci_dev *pdev,
> +						 const struct resource *res)
> +{
> +	return false;
> +}
> +
> +static inline int pci_tsm_retrieve_coherent_range(struct pci_dev *pdev,
> +						  struct range *out)
> +{
> +	return -ENODEV;
> +}
> +#endif /* CONFIG_PCI_TSM_CXL_COHERENT */
> +
>  #ifdef CONFIG_PCI_TSM
>  int pci_tsm_register(struct tsm_dev *tsm_dev);
>  void pci_tsm_unregister(struct tsm_dev *tsm_dev);
> -- 
> 2.53.0
>