Re: [PATCH v11 02/11] iommu/arm-smmu-v3: Make the ASID space per SMMU instance

[Mostafa Saleh <smostafa@google.com>]

On Mon, Oct 05, 2026 at 12:30:45PM -0700, Nicolin Chen wrote:
> An ASID tags the TLB entries within one SMMU, so two instances can use the
> same ASID without ever aliasing each other. Yet the driver allocates them
> out of a single global xarray, which makes the instances share a space that
> the hardware keeps apart, and lets one instance exhaust the IDs of another.
> 
> That global space is a leftover from the BTM support that shared ASIDs with
> the CPU. Now ARM_SMMU_FEAT_BTM is never set, nothing looks a domain up by
> its ASID, and a domain is pinned to one SMMU at the attach.
> 
> Give each SMMU its own asid_map, mirroring the per-SMMU vmid_map, and clean
> it up with devres, so that both of the ID maps get the same lifetime as the
> SMMU device structure that holds them. An upcoming change will need this,
> to reserve the crashed kernel's in-use ASIDs in the new map during a kdump
> kernel's stream table adoption.
> 
> This also eases the BTM work that Jason's working on:
> https://lore.kernel.org/linux-iommu/20261004162219.GA4064@nvidia.com/
> 
> Note that arm_smmu_asid_lock stays global, as it serializes the STE and CD
> updates against any ASID change rather than guarding the map itself, which
> does its own locking. Giving each SMMU its own lock looks possible now, but
> that would touch every attach path and belongs to a separate change.
> 
> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
> Tested-by: Breno Leitao <leitao@kernel.org>
> Tested-by: Cristian Prundeanu <cpru@amazon.com>
> Assisted-by: LLM
> Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>

Reviewed-by: Mostafa Saleh <smostafa@google.com>

Thanks,
Mostafa
> ---
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h    |  2 +-
>  .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c    |  6 +++---
>  drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c    | 18 +++++++++++++++---
>  3 files changed, 19 insertions(+), 7 deletions(-)
> 
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> index bb048e08d613d..8f2c0b1cc4a8d 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
> @@ -1003,6 +1003,7 @@ struct arm_smmu_device {
>  #define ARM_SMMU_MAX_VMIDS		(1 << 16)
>  	unsigned int			vmid_bits;
>  	struct ida			vmid_map;
> +	struct xarray			asid_map;
>  
>  	unsigned int			ssid_bits;
>  	unsigned int			sid_bits;
> @@ -1180,7 +1181,6 @@ to_smmu_nested_domain(struct iommu_domain *dom)
>  	return container_of(dom, struct arm_smmu_nested_domain, domain);
>  }
>  
> -extern struct xarray arm_smmu_asid_xa;
>  extern struct mutex arm_smmu_asid_lock;
>  
>  struct arm_smmu_domain *arm_smmu_domain_alloc(void);
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> index faeec656da9e7..489531e0027bd 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c
> @@ -311,7 +311,7 @@ static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
>  	 * reused, and if there is a race then it just suffers harmless
>  	 * unnecessary invalidation.
>  	 */
> -	xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
> +	xa_erase(&smmu_domain->smmu->asid_map, smmu_domain->cd.asid);
>  
>  	/*
>  	 * Actual free is defered to the SRCU callback
> @@ -352,7 +352,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
>  	smmu_domain->stage = ARM_SMMU_DOMAIN_SVA;
>  	smmu_domain->smmu = smmu;
>  
> -	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
> +	ret = xa_alloc(&smmu->asid_map, &asid, smmu_domain,
>  		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
>  	if (ret)
>  		goto err_free;
> @@ -366,7 +366,7 @@ struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
>  	return &smmu_domain->domain;
>  
>  err_asid:
> -	xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
> +	xa_erase(&smmu_domain->smmu->asid_map, smmu_domain->cd.asid);
>  err_free:
>  	arm_smmu_domain_free(smmu_domain);
>  	return ERR_PTR(ret);
> diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> index 792627e1c790e..aeace4335abfe 100644
> --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
> @@ -90,7 +90,6 @@ struct arm_smmu_option_prop {
>  	const char *prop;
>  };
>  
> -DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
>  DEFINE_MUTEX(arm_smmu_asid_lock);
>  
>  static struct arm_smmu_option_prop arm_smmu_options[] = {
> @@ -3015,7 +3014,7 @@ static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
>  	if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
>  		/* Prevent SVA from touching the CD while we're freeing it */
>  		mutex_lock(&arm_smmu_asid_lock);
> -		xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
> +		xa_erase(&smmu->asid_map, smmu_domain->cd.asid);
>  		mutex_unlock(&arm_smmu_asid_lock);
>  	} else {
>  		struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
> @@ -3035,7 +3034,7 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
>  
>  	/* Prevent SVA from modifying the ASID until it is written to the CD */
>  	mutex_lock(&arm_smmu_asid_lock);
> -	ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
> +	ret = xa_alloc(&smmu->asid_map, &asid, smmu_domain,
>  		       XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
>  	cd->asid	= (u16)asid;
>  	mutex_unlock(&arm_smmu_asid_lock);
> @@ -4741,6 +4740,13 @@ static void arm_smmu_destroy_vmid_map(void *data)
>  	ida_destroy(ida);
>  }
>  
> +static void arm_smmu_destroy_asid_map(void *data)
> +{
> +	struct xarray *xa = data;
> +
> +	xa_destroy(xa);
> +}
> +
>  static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
>  {
>  	int ret;
> @@ -4848,6 +4854,12 @@ static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
>  	if (ret)
>  		return ret;
>  
> +	xa_init_flags(&smmu->asid_map, XA_FLAGS_ALLOC1);
> +	ret = devm_add_action_or_reset(smmu->dev, arm_smmu_destroy_asid_map,
> +				       &smmu->asid_map);
> +	if (ret)
> +		return ret;
> +
>  	if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
>  		ret = arm_smmu_init_strtab_2lvl(smmu);
>  	else
> -- 
> 2.43.0
>