From: Guanghui Feng <guanghuifeng@linux.alibaba.com>
To: joro@8bytes.org, vasant.hegde@amd.com,
suravee.suthikulpanit@amd.com, will@kernel.org,
robin.murphy@arm.com
Cc: iommu@lists.linux.dev, linux-kernel@vger.kernel.org
Subject: [PATCH 2/2] iommu/amd: Reuse device data for IRTE cache invalidation
Date: Fri, 13 Nov 2026 17:50:17 +0800 [thread overview]
Message-ID: <20261113095017.1835921-3-guanghuifeng@linux.alibaba.com> (raw)
In-Reply-To: <20261113095017.1835921-1-guanghuifeng@linux.alibaba.com>
Switching an interrupt route between guest and remapping modes, changing
IRQ affinity, and enabling or disabling an IRTE all perform a linear
device lookup to invalidate the IRT cache for DMA aliases. Nested VM
entry and exit can repeat this for every posted interrupt route.
A reported workload with 4334 device entries shows median lookup
latencies of 22-33 us per invocation.
Save the allocation-time device data in struct amd_ir_data and thread it
through all IRTE update paths:
- modify_irte_ga() and modify_irte() accept a dev_data parameter and
pass it to iommu_flush_irt_and_complete().
- The amd_irte_ops callbacks (activate, deactivate, set_affinity) are
extended to carry dev_data, so amd_ir_update_irte() and the
irq_remapping activate/deactivate chip callbacks pass the cached
pointer.
- amd_iommu_activate_guest_mode() and deactivate_guest_mode() pass
ir_data->dev_data directly.
- free_irte() receives dev_data from irq_remapping_free().
- amd_ir_set_vcpu_affinity() reuses ir_data->dev_data for the vCPU
affinity capability check.
This eliminates all search_dev_data() list walks from IRTE update
paths. For non-PCI interrupt sources (IOAPIC, HPET) whose dev_data is
NULL from allocation, iommu_flush_irt_and_complete() skips the DMA
alias walk and invalidates only the single devid, which is correct
because these sources have no DMA aliases.
Device data is retained in the segment list across unplug and reused on
reprobe, so the cached pointer remains valid throughout the IRQ
lifetime. Continue resolving the PCI device through dev_data->dev so
that later device association is observed.
The invalidation path still skips everything when IRT caching is
disabled and preserves DMA alias invalidation, command ordering,
locking and completion waits when it is enabled.
Fixes: 5aac28784dca ("iommu/amd: Invalidate IRT cache for DMA aliases")
Reported-by: Yifei Ge <geyifei@linux.alibaba.com>
Co-developed-by: hr567 <hr567@linux.alibaba.com>
Signed-off-by: hr567 <hr567@linux.alibaba.com>
Signed-off-by: Guanghui Feng <guanghuifeng@linux.alibaba.com>
---
drivers/iommu/amd/amd_iommu_types.h | 10 ++--
drivers/iommu/amd/iommu.c | 74 ++++++++++++++++-------------
2 files changed, 48 insertions(+), 36 deletions(-)
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h
index f98770db0090..da6d90f3a574 100644
--- a/drivers/iommu/amd/amd_iommu_types.h
+++ b/drivers/iommu/amd/amd_iommu_types.h
@@ -1029,6 +1029,7 @@ struct irq_2_irte {
struct amd_ir_data {
struct amd_iommu *iommu;
+ struct iommu_dev_data *dev_data;
struct irq_2_irte irq_2_irte;
struct msi_msg msi_entry;
void *entry; /* Pointer to union irte or struct irte_ga */
@@ -1045,9 +1046,12 @@ struct amd_ir_data {
struct amd_irte_ops {
void (*prepare)(void *, u32, bool, u8, u32, int);
- void (*activate)(struct amd_iommu *iommu, void *, u16, u16);
- void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16);
- void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8, u32);
+ void (*activate)(struct amd_iommu *iommu, void *, u16, u16,
+ struct iommu_dev_data *);
+ void (*deactivate)(struct amd_iommu *iommu, void *, u16, u16,
+ struct iommu_dev_data *);
+ void (*set_affinity)(struct amd_iommu *iommu, void *, u16, u16, u8,
+ u32, struct iommu_dev_data *);
void *(*get)(struct irq_remap_table *, int);
void (*set_allocated)(struct irq_remap_table *, int);
bool (*is_allocated)(struct irq_remap_table *, int);
diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c
index 8f5a38ae2e29..6876de9cce71 100644
--- a/drivers/iommu/amd/iommu.c
+++ b/drivers/iommu/amd/iommu.c
@@ -3291,19 +3291,18 @@ static int iommu_flush_dev_irt(struct pci_dev *unused, u16 devid, void *data)
return ret;
}
-static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid)
+static void iommu_flush_irt_and_complete(struct amd_iommu *iommu, u16 devid,
+ struct iommu_dev_data *dev_data)
{
int ret;
u64 data;
unsigned long flags;
struct iommu_cmd cmd;
struct pci_dev *pdev = NULL;
- struct iommu_dev_data *dev_data;
if (iommu->irtcachedis_enabled)
return;
- dev_data = search_dev_data(iommu, devid);
if (dev_data && dev_data->dev && dev_is_pci(dev_data->dev))
pdev = to_pci_dev(dev_data->dev);
@@ -3579,7 +3578,7 @@ static int __modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
}
static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
- struct irte_ga *irte)
+ struct irte_ga *irte, struct iommu_dev_data *dev_data)
{
int ret;
@@ -3587,13 +3586,13 @@ static int modify_irte_ga(struct amd_iommu *iommu, u16 devid, int index,
if (ret)
return ret;
- iommu_flush_irt_and_complete(iommu, devid);
+ iommu_flush_irt_and_complete(iommu, devid, dev_data);
return 0;
}
-static int modify_irte(struct amd_iommu *iommu,
- u16 devid, int index, union irte *irte)
+static int modify_irte(struct amd_iommu *iommu, u16 devid, int index,
+ union irte *irte, struct iommu_dev_data *dev_data)
{
struct irq_remap_table *table;
unsigned long flags;
@@ -3606,12 +3605,13 @@ static int modify_irte(struct amd_iommu *iommu,
table->table[index] = irte->val;
raw_spin_unlock_irqrestore(&table->lock, flags);
- iommu_flush_irt_and_complete(iommu, devid);
+ iommu_flush_irt_and_complete(iommu, devid, dev_data);
return 0;
}
-static void free_irte(struct amd_iommu *iommu, u16 devid, int index)
+static void free_irte(struct amd_iommu *iommu, u16 devid, int index,
+ struct iommu_dev_data *dev_data)
{
struct irq_remap_table *table;
unsigned long flags;
@@ -3624,7 +3624,7 @@ static void free_irte(struct amd_iommu *iommu, u16 devid, int index)
iommu->irte_ops->clear_allocated(table, index);
raw_spin_unlock_irqrestore(&table->lock, flags);
- iommu_flush_irt_and_complete(iommu, devid);
+ iommu_flush_irt_and_complete(iommu, devid, dev_data);
}
static void irte_prepare(void *entry,
@@ -3657,50 +3657,57 @@ static void irte_ga_prepare(void *entry,
irte->lo.fields_remap.valid = 1;
}
-static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
+static void irte_activate(struct amd_iommu *iommu, void *entry, u16 devid,
+ u16 index, struct iommu_dev_data *dev_data)
{
union irte *irte = (union irte *) entry;
irte->fields.valid = 1;
- modify_irte(iommu, devid, index, irte);
+ modify_irte(iommu, devid, index, irte, dev_data);
}
-static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
+static void irte_ga_activate(struct amd_iommu *iommu, void *entry, u16 devid,
+ u16 index, struct iommu_dev_data *dev_data)
{
struct irte_ga *irte = (struct irte_ga *) entry;
irte->lo.fields_remap.valid = 1;
- modify_irte_ga(iommu, devid, index, irte);
+ modify_irte_ga(iommu, devid, index, irte, dev_data);
}
-static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
+static void irte_deactivate(struct amd_iommu *iommu, void *entry, u16 devid,
+ u16 index, struct iommu_dev_data *dev_data)
{
union irte *irte = (union irte *) entry;
irte->fields.valid = 0;
- modify_irte(iommu, devid, index, irte);
+ modify_irte(iommu, devid, index, irte, dev_data);
}
-static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid, u16 index)
+static void irte_ga_deactivate(struct amd_iommu *iommu, void *entry, u16 devid,
+ u16 index, struct iommu_dev_data *dev_data)
{
struct irte_ga *irte = (struct irte_ga *) entry;
irte->lo.fields_remap.valid = 0;
- modify_irte_ga(iommu, devid, index, irte);
+ modify_irte_ga(iommu, devid, index, irte, dev_data);
}
-static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
- u8 vector, u32 dest_apicid)
+static void irte_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid,
+ u16 index, u8 vector, u32 dest_apicid,
+ struct iommu_dev_data *dev_data)
{
union irte *irte = (union irte *) entry;
irte->fields.vector = vector;
irte->fields.destination = dest_apicid;
- modify_irte(iommu, devid, index, irte);
+ modify_irte(iommu, devid, index, irte, dev_data);
}
-static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid, u16 index,
- u8 vector, u32 dest_apicid)
+static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry,
+ u16 devid, u16 index, u8 vector,
+ u32 dest_apicid,
+ struct iommu_dev_data *dev_data)
{
struct irte_ga *irte = (struct irte_ga *) entry;
@@ -3710,7 +3717,7 @@ static void irte_ga_set_affinity(struct amd_iommu *iommu, void *entry, u16 devid
APICID_TO_IRTE_DEST_LO(dest_apicid);
irte->hi.fields.destination =
APICID_TO_IRTE_DEST_HI(dest_apicid);
- modify_irte_ga(iommu, devid, index, irte);
+ modify_irte_ga(iommu, devid, index, irte, dev_data);
}
}
@@ -3946,6 +3953,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
}
data->iommu = iommu;
+ data->dev_data = dev_data;
irq_data->hwirq = (devid << 16) + i;
irq_data->chip_data = data;
irq_data->chip = &amd_ir_chip;
@@ -3964,7 +3972,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,
}
}
for (i = 0; i < nr_irqs; i++)
- free_irte(iommu, devid, index + i);
+ free_irte(iommu, devid, index + i, dev_data);
out_free_parent:
irq_domain_free_irqs_common(domain, virq, nr_irqs);
return ret;
@@ -3983,7 +3991,8 @@ static void irq_remapping_free(struct irq_domain *domain, unsigned int virq,
if (irq_data && irq_data->chip_data) {
data = irq_data->chip_data;
irte_info = &data->irq_2_irte;
- free_irte(data->iommu, irte_info->devid, irte_info->index);
+ free_irte(data->iommu, irte_info->devid,
+ irte_info->index, data->dev_data);
kfree(data->entry);
kfree(data);
}
@@ -4008,7 +4017,7 @@ static int irq_remapping_activate(struct irq_domain *domain,
return 0;
iommu->irte_ops->activate(iommu, data->entry, irte_info->devid,
- irte_info->index);
+ irte_info->index, data->dev_data);
amd_ir_update_irte(irq_data, iommu, data, irte_info, cfg);
return 0;
}
@@ -4022,7 +4031,7 @@ static void irq_remapping_deactivate(struct irq_domain *domain,
if (iommu)
iommu->irte_ops->deactivate(iommu, data->entry, irte_info->devid,
- irte_info->index);
+ irte_info->index, data->dev_data);
}
static int irq_remapping_select(struct irq_domain *d, struct irq_fwspec *fwspec,
@@ -4134,7 +4143,7 @@ int amd_iommu_activate_guest_mode(void *data, int cpu, bool ga_log_intr)
__amd_iommu_update_ga(entry, cpu, ga_log_intr);
return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
- ir_data->irq_2_irte.index, entry);
+ ir_data->irq_2_irte.index, entry, ir_data->dev_data);
}
EXPORT_SYMBOL(amd_iommu_activate_guest_mode);
@@ -4166,7 +4175,7 @@ int amd_iommu_deactivate_guest_mode(void *data)
APICID_TO_IRTE_DEST_HI(cfg->dest_apicid);
return modify_irte_ga(ir_data->iommu, ir_data->irq_2_irte.devid,
- ir_data->irq_2_irte.index, entry);
+ ir_data->irq_2_irte.index, entry, ir_data->dev_data);
}
EXPORT_SYMBOL(amd_iommu_deactivate_guest_mode);
@@ -4175,7 +4184,6 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *info)
int ret;
struct amd_iommu_pi_data *pi_data = info;
struct amd_ir_data *ir_data = data->chip_data;
- struct irq_2_irte *irte_info = &ir_data->irq_2_irte;
struct iommu_dev_data *dev_data;
if (WARN_ON_ONCE(!AMD_IOMMU_GUEST_IR_VAPIC(amd_iommu_guest_ir)))
@@ -4184,7 +4192,7 @@ static int amd_ir_set_vcpu_affinity(struct irq_data *data, void *info)
if (ir_data->iommu == NULL)
return -EINVAL;
- dev_data = search_dev_data(ir_data->iommu, irte_info->devid);
+ dev_data = ir_data->dev_data;
/* Note:
* This device has never been set up for guest mode.
@@ -4226,7 +4234,7 @@ static void amd_ir_update_irte(struct irq_data *irqd, struct amd_iommu *iommu,
*/
iommu->irte_ops->set_affinity(iommu, ir_data->entry, irte_info->devid,
irte_info->index, cfg->vector,
- cfg->dest_apicid);
+ cfg->dest_apicid, ir_data->dev_data);
}
static int amd_ir_set_affinity(struct irq_data *data,
--
2.43.7
prev parent reply other threads:[~2026-11-13 9:50 UTC|newest]
Thread overview: 3+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-13 9:50 [PATCH 0/2] iommu/amd: Optimize IRT cache invalidation for nested VM workloads Guanghui Feng
2026-11-13 9:50 ` [PATCH 1/2] iommu/amd: Skip device lookup when IRT caching is disabled Guanghui Feng
2026-11-13 9:50 ` Guanghui Feng [this message]
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261113095017.1835921-3-guanghuifeng@linux.alibaba.com \
--to=guanghuifeng@linux.alibaba.com \
--cc=iommu@lists.linux.dev \
--cc=joro@8bytes.org \
--cc=linux-kernel@vger.kernel.org \
--cc=robin.murphy@arm.com \
--cc=suravee.suthikulpanit@amd.com \
--cc=vasant.hegde@amd.com \
--cc=will@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®