From: Nicolin Chen <nicolinc@nvidia.com>
To: <jgg@nvidia.com>, <kevin.tian@intel.com>, <will@kernel.org>
Cc: <joro@8bytes.org>, <suravee.suthikulpanit@amd.com>,
<robin.murphy@arm.com>, <dwmw2@infradead.org>,
<baolu.lu@linux.intel.com>, <shuah@kernel.org>,
<linux-kernel@vger.kernel.org>, <iommu@lists.linux.dev>,
<linux-arm-kernel@lists.infradead.org>,
<linux-kselftest@vger.kernel.org>, <eric.auger@redhat.com>,
<jean-philippe@linaro.org>, <mdf@kernel.org>,
<mshavit@google.com>, <shameerali.kolothum.thodi@huawei.com>,
<smostafa@google.com>, <yi.l.liu@intel.com>
Subject: [PATCH v2 16/19] iommu/arm-smmu-v3: Add arm_smmu_cache_invalidate_user
Date: Tue, 27 Aug 2024 09:59:53 -0700 [thread overview]
Message-ID: <3548bfff43a0e1c072b77fc37a1a8a6c930bcec9.1724776335.git.nicolinc@nvidia.com> (raw)
In-Reply-To: <cover.1724776335.git.nicolinc@nvidia.com>
Add arm_smmu_cache_invalidate_user() function for user space to invalidate
IOTLB entries that are still cached by the hardware.
Add struct iommu_hwpt_arm_smmuv3_invalidate defining an invalidation entry
that is simply the native format of a 128-bit TLBI command. Scan commands
against the permitted command list and fix their VMID fields.
Co-developed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Co-developed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 115 ++++++++++++++++++++
drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 1 +
include/uapi/linux/iommufd.h | 21 ++++
3 files changed, 137 insertions(+)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 6d40f1e150cb..a2af693bc7b2 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3267,10 +3267,117 @@ static void arm_smmu_domain_nested_free(struct iommu_domain *domain)
kfree(container_of(domain, struct arm_smmu_nested_domain, domain));
}
+/*
+ * Convert, in place, the raw invalidation command into an internal format that
+ * can be passed to arm_smmu_cmdq_issue_cmdlist(). Internally commands are
+ * stored in CPU endian.
+ *
+ * Enforce the VMID on the command.
+ */
+static int
+arm_smmu_convert_user_cmd(struct arm_smmu_domain *s2_parent,
+ struct iommu_hwpt_arm_smmuv3_invalidate *cmd)
+{
+ u16 vmid = s2_parent->s2_cfg.vmid;
+
+ cmd->cmd[0] = le64_to_cpu(cmd->cmd[0]);
+ cmd->cmd[1] = le64_to_cpu(cmd->cmd[1]);
+
+ switch (cmd->cmd[0] & CMDQ_0_OP) {
+ case CMDQ_OP_TLBI_NSNH_ALL:
+ /* Convert to NH_ALL */
+ cmd->cmd[0] = CMDQ_OP_TLBI_NH_ALL |
+ FIELD_PREP(CMDQ_TLBI_0_VMID, vmid);
+ cmd->cmd[1] = 0;
+ break;
+ case CMDQ_OP_TLBI_NH_VA:
+ case CMDQ_OP_TLBI_NH_VAA:
+ case CMDQ_OP_TLBI_NH_ALL:
+ case CMDQ_OP_TLBI_NH_ASID:
+ cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID;
+ cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vmid);
+ break;
+ default:
+ return -EIO;
+ }
+ return 0;
+}
+
+static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent,
+ struct iommu_user_data_array *array)
+{
+ struct arm_smmu_device *smmu = s2_parent->smmu;
+ struct iommu_hwpt_arm_smmuv3_invalidate *last_batch;
+ struct iommu_hwpt_arm_smmuv3_invalidate *cmds;
+ struct iommu_hwpt_arm_smmuv3_invalidate *cur;
+ struct iommu_hwpt_arm_smmuv3_invalidate *end;
+ struct arm_smmu_cmdq_ent ent;
+ struct arm_smmu_cmdq *cmdq;
+ int ret;
+
+ /* A zero-length array is allowed to validate the array type */
+ if (array->entry_num == 0 &&
+ array->type == IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3) {
+ array->entry_num = 0;
+ return 0;
+ }
+
+ cmds = kcalloc(array->entry_num, sizeof(*cmds), GFP_KERNEL);
+ if (!cmds)
+ return -ENOMEM;
+ cur = cmds;
+ end = cmds + array->entry_num;
+
+ static_assert(sizeof(*cmds) == 2 * sizeof(u64));
+ ret = iommu_copy_struct_from_full_user_array(
+ cmds, sizeof(*cmds), array,
+ IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3);
+ if (ret)
+ goto out;
+
+ ent.opcode = cmds->cmd[0] & CMDQ_0_OP;
+ cmdq = arm_smmu_get_cmdq(smmu, &ent);
+
+ last_batch = cmds;
+ while (cur != end) {
+ ret = arm_smmu_convert_user_cmd(s2_parent, cur);
+ if (ret)
+ goto out;
+
+ /* FIXME work in blocks of CMDQ_BATCH_ENTRIES and copy each block? */
+ cur++;
+ if (cur != end && (cur - last_batch) != CMDQ_BATCH_ENTRIES - 1)
+ continue;
+
+ ret = arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, last_batch->cmd,
+ cur - last_batch, true);
+ if (ret) {
+ cur--;
+ goto out;
+ }
+ last_batch = cur;
+ }
+out:
+ array->entry_num = cur - cmds;
+ kfree(cmds);
+ return ret;
+}
+
+static int arm_smmu_cache_invalidate_user(struct iommu_domain *domain,
+ struct iommu_user_data_array *array)
+{
+ struct arm_smmu_nested_domain *nested_domain =
+ container_of(domain, struct arm_smmu_nested_domain, domain);
+
+ return __arm_smmu_cache_invalidate_user(
+ nested_domain->s2_parent, array);
+}
+
static const struct iommu_domain_ops arm_smmu_nested_ops = {
.get_msi_mapping_domain = arm_smmu_get_msi_mapping_domain,
.attach_dev = arm_smmu_attach_dev_nested,
.free = arm_smmu_domain_nested_free,
+ .cache_invalidate_user = arm_smmu_cache_invalidate_user,
};
static struct iommu_domain *
@@ -3298,6 +3405,14 @@ arm_smmu_domain_alloc_nesting(struct device *dev, u32 flags,
!(master->smmu->features & ARM_SMMU_FEAT_S2FWB))
return ERR_PTR(-EOPNOTSUPP);
+ /*
+ * FORCE_SYNC is not set with FEAT_NESTING. Some study of the exact HW
+ * defect is needed to determine if arm_smmu_cache_invalidate_user()
+ * needs any change to remove this.
+ */
+ if (WARN_ON(master->smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC))
+ return ERR_PTR(-EOPNOTSUPP);
+
ret = iommu_copy_struct_from_user(&arg, user_data,
IOMMU_HWPT_DATA_ARM_SMMUV3, ste);
if (ret)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 79afaef18906..6c8ae70c90fe 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -523,6 +523,7 @@ struct arm_smmu_cmdq_ent {
#define CMDQ_OP_TLBI_NH_ALL 0x10
#define CMDQ_OP_TLBI_NH_ASID 0x11
#define CMDQ_OP_TLBI_NH_VA 0x12
+ #define CMDQ_OP_TLBI_NH_VAA 0x13
#define CMDQ_OP_TLBI_EL2_ALL 0x20
#define CMDQ_OP_TLBI_EL2_ASID 0x21
#define CMDQ_OP_TLBI_EL2_VA 0x22
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index fd7d16fd441d..f3aefb11f681 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -685,9 +685,11 @@ struct iommu_hwpt_get_dirty_bitmap {
* enum iommu_hwpt_invalidate_data_type - IOMMU HWPT Cache Invalidation
* Data Type
* @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1
+ * @IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3: Invalidation data for ARM SMMUv3
*/
enum iommu_hwpt_invalidate_data_type {
IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
+ IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3 = 1,
};
/**
@@ -726,6 +728,25 @@ struct iommu_hwpt_vtd_s1_invalidate {
__u32 __reserved;
};
+/**
+ * struct iommu_hwpt_arm_smmuv3_invalidate - ARM SMMUv3 cahce invalidation
+ * (IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3)
+ * @cmd: 128-bit cache invalidation command that runs in SMMU CMDQ.
+ * Must be little-endian.
+ *
+ * Supported command list:
+ * CMDQ_OP_TLBI_NSNH_ALL
+ * CMDQ_OP_TLBI_NH_VA
+ * CMDQ_OP_TLBI_NH_VAA
+ * CMDQ_OP_TLBI_NH_ALL
+ * CMDQ_OP_TLBI_NH_ASID
+ *
+ * -EIO will be returned if the command is not supported.
+ */
+struct iommu_hwpt_arm_smmuv3_invalidate {
+ __aligned_u64 cmd[2];
+};
+
/**
* struct iommu_hwpt_invalidate - ioctl(IOMMU_HWPT_INVALIDATE)
* @size: sizeof(struct iommu_hwpt_invalidate)
--
2.43.0
next prev parent reply other threads:[~2024-08-27 17:02 UTC|newest]
Thread overview: 149+ messages / expand[flat|nested] mbox.gz Atom feed top
2024-08-27 16:59 [PATCH v2 00/19] iommufd: Add VIOMMU infrastructure (Part-1) Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 01/19] iommufd: Reorder struct forward declarations Nicolin Chen
2024-09-05 15:44 ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 02/19] iommufd/viommu: Add IOMMUFD_OBJ_VIOMMU and IOMMU_VIOMMU_ALLOC ioctl Nicolin Chen
2024-09-01 2:39 ` Baolu Lu
2024-09-02 5:27 ` Nicolin Chen
2024-09-04 16:26 ` Jason Gunthorpe
2024-09-04 17:29 ` Nicolin Chen
2024-09-04 23:37 ` Jason Gunthorpe
2024-09-05 3:07 ` Nicolin Chen
2024-09-12 3:39 ` Nicolin Chen
2024-09-15 21:10 ` Jason Gunthorpe
2024-09-05 15:53 ` Jason Gunthorpe
2024-09-05 17:10 ` Nicolin Chen
2024-09-05 17:41 ` Jason Gunthorpe
2024-09-05 18:04 ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 03/19] iommu: Pass in a viommu pointer to domain_alloc_user op Nicolin Chen
2024-09-05 15:54 ` Jason Gunthorpe
2024-09-05 17:19 ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 04/19] iommufd: Allow pt_id to carry viommu_id for IOMMU_HWPT_ALLOC Nicolin Chen
2024-09-05 15:56 ` Jason Gunthorpe
2024-09-26 8:50 ` Yi Liu
2024-09-26 20:10 ` Nicolin Chen
2024-09-27 0:43 ` Tian, Kevin
2024-09-27 1:25 ` Nicolin Chen
2024-09-27 2:23 ` Tian, Kevin
2024-09-27 2:57 ` Nicolin Chen
2024-09-27 5:38 ` Yi Liu
2024-09-27 6:02 ` Nicolin Chen
2024-09-27 11:59 ` Jason Gunthorpe
2024-09-27 19:52 ` Nicolin Chen
2024-09-27 12:12 ` Yi Liu
2024-09-27 20:29 ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 05/19] iommufd/selftest: Add IOMMU_VIOMMU_ALLOC test coverage Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 06/19] iommufd/viommu: Add IOMMU_VIOMMU_SET/UNSET_VDEV_ID ioctl Nicolin Chen
2024-09-05 16:03 ` Jason Gunthorpe
2024-09-05 17:37 ` Nicolin Chen
2024-09-05 17:43 ` Jason Gunthorpe
2024-09-05 20:14 ` Nicolin Chen
2024-09-11 6:19 ` Tian, Kevin
2024-09-11 7:11 ` Nicolin Chen
2024-09-11 7:18 ` Tian, Kevin
2024-09-11 7:47 ` Nicolin Chen
2024-10-01 8:54 ` Nicolin Chen
2024-10-01 13:46 ` Jason Gunthorpe
2024-10-01 18:45 ` Nicolin Chen
2024-09-27 13:50 ` Mostafa Saleh
2024-09-27 14:01 ` Jason Gunthorpe
2024-09-27 14:22 ` Mostafa Saleh
2024-09-27 14:58 ` Jason Gunthorpe
2024-09-27 15:59 ` Mostafa Saleh
2024-10-04 4:32 ` Alexey Kardashevskiy
2024-10-04 5:33 ` Nicolin Chen
2024-10-04 11:41 ` Jason Gunthorpe
2024-10-04 18:13 ` Nicolin Chen
2024-10-04 18:50 ` Jason Gunthorpe
2024-10-04 19:25 ` Nicolin Chen
2024-10-04 20:17 ` Jason Gunthorpe
2024-10-04 20:33 ` Nicolin Chen
2024-10-07 17:18 ` Jason Gunthorpe
2025-04-30 7:54 ` Alexey Kardashevskiy
2025-04-30 7:58 ` Nicolin Chen
2025-05-05 17:08 ` Jason Gunthorpe
2025-05-06 2:53 ` Nicolin Chen
2025-05-06 4:35 ` Alexey Kardashevskiy
2025-05-06 4:46 ` Tian, Kevin
2025-05-06 12:58 ` Jason Gunthorpe
2025-05-06 19:55 ` Nicolin Chen
2025-05-07 12:28 ` Jason Gunthorpe
2025-05-07 11:18 ` Alexey Kardashevskiy
2025-05-07 12:24 ` Jason Gunthorpe
2025-05-09 2:57 ` Alexey Kardashevskiy
2025-05-09 22:07 ` Jason Gunthorpe
2025-05-10 12:51 ` Alexey Kardashevskiy
2025-05-12 15:31 ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 07/19] iommufd/selftest: Add IOMMU_VIOMMU_SET/UNSET_VDEV_ID test coverage Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 08/19] iommufd/viommu: Add cache_invalidate for IOMMU_VIOMMU_TYPE_DEFAULT Nicolin Chen
2024-09-05 16:05 ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 09/19] iommufd: Allow hwpt_id to carry viommu_id for IOMMU_HWPT_INVALIDATE Nicolin Chen
2024-09-05 16:10 ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 10/19] iommufd/viommu: Add vdev_id helpers for IOMMU drivers Nicolin Chen
2024-09-05 16:14 ` Jason Gunthorpe
2024-09-05 17:53 ` Nicolin Chen
2024-09-11 23:11 ` Jason Gunthorpe
2024-09-12 3:17 ` Nicolin Chen
2024-10-05 5:19 ` Nicolin Chen
2024-10-07 15:38 ` Jason Gunthorpe
2024-10-07 16:36 ` Nicolin Chen
2024-10-07 17:11 ` Jason Gunthorpe
2024-10-07 17:25 ` Nicolin Chen
2024-10-07 18:28 ` Jason Gunthorpe
2024-10-07 19:56 ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 11/19] iommu: Add iommu_copy_struct_from_full_user_array helper Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 12/19] iommufd/selftest: Add mock_viommu_invalidate_user op Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 13/19] iommufd/selftest: Add IOMMU_TEST_OP_DEV_CHECK_CACHE test command Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 14/19] iommufd/selftest: Add VIOMMU coverage for IOMMU_HWPT_INVALIDATE ioctl Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 15/19] iommufd/viommu: Add iommufd_viommu_to_parent_domain helper Nicolin Chen
2024-08-27 16:59 ` Nicolin Chen [this message]
2024-09-05 16:23 ` [PATCH v2 16/19] iommu/arm-smmu-v3: Add arm_smmu_cache_invalidate_user Jason Gunthorpe
2024-09-05 17:56 ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 17/19] iommu/arm-smmu-v3: Add arm_smmu_viommu_cache_invalidate Nicolin Chen
2024-09-05 16:20 ` Jason Gunthorpe
2024-09-05 18:00 ` Nicolin Chen
2024-09-05 18:21 ` Jason Gunthorpe
2024-09-11 6:25 ` Tian, Kevin
2024-09-11 7:20 ` Nicolin Chen
2024-09-11 7:50 ` Baolu Lu
2024-09-11 8:17 ` Tian, Kevin
2024-09-11 8:19 ` Baolu Lu
2024-09-11 21:08 ` Nicolin Chen
2024-09-12 4:45 ` Baolu Lu
2024-09-11 8:13 ` Tian, Kevin
2024-09-11 20:53 ` Nicolin Chen
2024-09-11 23:07 ` Jason Gunthorpe
2024-09-13 2:33 ` Tian, Kevin
2024-09-14 14:50 ` Jason Gunthorpe
2024-09-18 8:10 ` Tian, Kevin
2024-09-23 18:34 ` Jason Gunthorpe
2024-09-11 23:00 ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 18/19] iommu/arm-smmu-v3: Allow ATS for IOMMU_DOMAIN_NESTED Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 19/19] iommu/arm-smmu-v3: Update comments about ATS and bypass Nicolin Chen
2024-09-11 6:12 ` [PATCH v2 00/19] iommufd: Add VIOMMU infrastructure (Part-1) Tian, Kevin
2024-09-11 7:08 ` Nicolin Chen
2024-09-11 7:18 ` Tian, Kevin
2024-09-11 7:40 ` Nicolin Chen
2024-09-11 8:08 ` Tian, Kevin
2024-09-11 20:21 ` Nicolin Chen
2024-10-01 1:55 ` Alexey Kardashevskiy
2024-10-01 3:36 ` Nicolin Chen
2024-10-01 5:06 ` Alexey Kardashevskiy
2024-10-01 13:44 ` Jason Gunthorpe
2024-09-25 10:30 ` Yi Liu
2024-09-25 18:55 ` Nicolin Chen
2024-09-26 8:47 ` Yi Liu
2024-09-26 20:03 ` Nicolin Chen
2024-09-27 2:05 ` Baolu Lu
2024-09-27 6:14 ` Yi Liu
2024-09-27 5:54 ` Yi Liu
2024-09-27 6:32 ` Nicolin Chen
2024-09-27 12:12 ` Yi Liu
2024-09-27 12:20 ` Jason Gunthorpe
2024-09-29 7:19 ` Yi Liu
2024-10-01 13:48 ` Jason Gunthorpe
2024-10-01 18:40 ` Nicolin Chen
2024-09-27 20:44 ` Nicolin Chen
2024-09-29 7:16 ` Yi Liu
2024-09-30 21:59 ` Nicolin Chen
2024-10-09 7:20 ` Yi Liu
2024-10-09 12:31 ` Jason Gunthorpe
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=3548bfff43a0e1c072b77fc37a1a8a6c930bcec9.1724776335.git.nicolinc@nvidia.com \
--to=nicolinc@nvidia.com \
--cc=baolu.lu@linux.intel.com \
--cc=dwmw2@infradead.org \
--cc=eric.auger@redhat.com \
--cc=iommu@lists.linux.dev \
--cc=jean-philippe@linaro.org \
--cc=jgg@nvidia.com \
--cc=joro@8bytes.org \
--cc=kevin.tian@intel.com \
--cc=linux-arm-kernel@lists.infradead.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-kselftest@vger.kernel.org \
--cc=mdf@kernel.org \
--cc=mshavit@google.com \
--cc=robin.murphy@arm.com \
--cc=shameerali.kolothum.thodi@huawei.com \
--cc=shuah@kernel.org \
--cc=smostafa@google.com \
--cc=suravee.suthikulpanit@amd.com \
--cc=will@kernel.org \
--cc=yi.l.liu@intel.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®