mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Nicolin Chen <nicolinc@nvidia.com>
To: <jgg@nvidia.com>, <kevin.tian@intel.com>, <will@kernel.org>
Cc: <joro@8bytes.org>, <suravee.suthikulpanit@amd.com>,
	<robin.murphy@arm.com>, <dwmw2@infradead.org>,
	<baolu.lu@linux.intel.com>, <shuah@kernel.org>,
	<linux-kernel@vger.kernel.org>, <iommu@lists.linux.dev>,
	<linux-arm-kernel@lists.infradead.org>,
	<linux-kselftest@vger.kernel.org>, <eric.auger@redhat.com>,
	<jean-philippe@linaro.org>, <mdf@kernel.org>,
	<mshavit@google.com>, <shameerali.kolothum.thodi@huawei.com>,
	<smostafa@google.com>, <yi.l.liu@intel.com>
Subject: [PATCH v2 16/19] iommu/arm-smmu-v3: Add arm_smmu_cache_invalidate_user
Date: Tue, 27 Aug 2024 09:59:53 -0700	[thread overview]
Message-ID: <3548bfff43a0e1c072b77fc37a1a8a6c930bcec9.1724776335.git.nicolinc@nvidia.com> (raw)
In-Reply-To: <cover.1724776335.git.nicolinc@nvidia.com>

Add arm_smmu_cache_invalidate_user() function for user space to invalidate
IOTLB entries that are still cached by the hardware.

Add struct iommu_hwpt_arm_smmuv3_invalidate defining an invalidation entry
that is simply the native format of a 128-bit TLBI command. Scan commands
against the permitted command list and fix their VMID fields.

Co-developed-by: Eric Auger <eric.auger@redhat.com>
Signed-off-by: Eric Auger <eric.auger@redhat.com>
Co-developed-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
Signed-off-by: Nicolin Chen <nicolinc@nvidia.com>
---
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 115 ++++++++++++++++++++
 drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h |   1 +
 include/uapi/linux/iommufd.h                |  21 ++++
 3 files changed, 137 insertions(+)

diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
index 6d40f1e150cb..a2af693bc7b2 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c
@@ -3267,10 +3267,117 @@ static void arm_smmu_domain_nested_free(struct iommu_domain *domain)
 	kfree(container_of(domain, struct arm_smmu_nested_domain, domain));
 }
 
+/*
+ * Convert, in place, the raw invalidation command into an internal format that
+ * can be passed to arm_smmu_cmdq_issue_cmdlist(). Internally commands are
+ * stored in CPU endian.
+ *
+ * Enforce the VMID on the command.
+ */
+static int
+arm_smmu_convert_user_cmd(struct arm_smmu_domain *s2_parent,
+			  struct iommu_hwpt_arm_smmuv3_invalidate *cmd)
+{
+	u16 vmid = s2_parent->s2_cfg.vmid;
+
+	cmd->cmd[0] = le64_to_cpu(cmd->cmd[0]);
+	cmd->cmd[1] = le64_to_cpu(cmd->cmd[1]);
+
+	switch (cmd->cmd[0] & CMDQ_0_OP) {
+	case CMDQ_OP_TLBI_NSNH_ALL:
+		/* Convert to NH_ALL */
+		cmd->cmd[0] = CMDQ_OP_TLBI_NH_ALL |
+			      FIELD_PREP(CMDQ_TLBI_0_VMID, vmid);
+		cmd->cmd[1] = 0;
+		break;
+	case CMDQ_OP_TLBI_NH_VA:
+	case CMDQ_OP_TLBI_NH_VAA:
+	case CMDQ_OP_TLBI_NH_ALL:
+	case CMDQ_OP_TLBI_NH_ASID:
+		cmd->cmd[0] &= ~CMDQ_TLBI_0_VMID;
+		cmd->cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, vmid);
+		break;
+	default:
+		return -EIO;
+	}
+	return 0;
+}
+
+static int __arm_smmu_cache_invalidate_user(struct arm_smmu_domain *s2_parent,
+					    struct iommu_user_data_array *array)
+{
+	struct arm_smmu_device *smmu = s2_parent->smmu;
+	struct iommu_hwpt_arm_smmuv3_invalidate *last_batch;
+	struct iommu_hwpt_arm_smmuv3_invalidate *cmds;
+	struct iommu_hwpt_arm_smmuv3_invalidate *cur;
+	struct iommu_hwpt_arm_smmuv3_invalidate *end;
+	struct arm_smmu_cmdq_ent ent;
+	struct arm_smmu_cmdq *cmdq;
+	int ret;
+
+	/* A zero-length array is allowed to validate the array type */
+	if (array->entry_num == 0 &&
+	    array->type == IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3) {
+		array->entry_num = 0;
+		return 0;
+	}
+
+	cmds = kcalloc(array->entry_num, sizeof(*cmds), GFP_KERNEL);
+	if (!cmds)
+		return -ENOMEM;
+	cur = cmds;
+	end = cmds + array->entry_num;
+
+	static_assert(sizeof(*cmds) == 2 * sizeof(u64));
+	ret = iommu_copy_struct_from_full_user_array(
+		cmds, sizeof(*cmds), array,
+		IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3);
+	if (ret)
+		goto out;
+
+	ent.opcode = cmds->cmd[0] & CMDQ_0_OP;
+	cmdq = arm_smmu_get_cmdq(smmu, &ent);
+
+	last_batch = cmds;
+	while (cur != end) {
+		ret = arm_smmu_convert_user_cmd(s2_parent, cur);
+		if (ret)
+			goto out;
+
+		/* FIXME work in blocks of CMDQ_BATCH_ENTRIES and copy each block? */
+		cur++;
+		if (cur != end && (cur - last_batch) != CMDQ_BATCH_ENTRIES - 1)
+			continue;
+
+		ret = arm_smmu_cmdq_issue_cmdlist(smmu, cmdq, last_batch->cmd,
+						  cur - last_batch, true);
+		if (ret) {
+			cur--;
+			goto out;
+		}
+		last_batch = cur;
+	}
+out:
+	array->entry_num = cur - cmds;
+	kfree(cmds);
+	return ret;
+}
+
+static int arm_smmu_cache_invalidate_user(struct iommu_domain *domain,
+					  struct iommu_user_data_array *array)
+{
+	struct arm_smmu_nested_domain *nested_domain =
+		container_of(domain, struct arm_smmu_nested_domain, domain);
+
+	return __arm_smmu_cache_invalidate_user(
+			nested_domain->s2_parent, array);
+}
+
 static const struct iommu_domain_ops arm_smmu_nested_ops = {
 	.get_msi_mapping_domain	= arm_smmu_get_msi_mapping_domain,
 	.attach_dev = arm_smmu_attach_dev_nested,
 	.free = arm_smmu_domain_nested_free,
+	.cache_invalidate_user	= arm_smmu_cache_invalidate_user,
 };
 
 static struct iommu_domain *
@@ -3298,6 +3405,14 @@ arm_smmu_domain_alloc_nesting(struct device *dev, u32 flags,
 	    !(master->smmu->features & ARM_SMMU_FEAT_S2FWB))
 		return ERR_PTR(-EOPNOTSUPP);
 
+	/*
+	 * FORCE_SYNC is not set with FEAT_NESTING. Some study of the exact HW
+	 * defect is needed to determine if arm_smmu_cache_invalidate_user()
+	 * needs any change to remove this.
+	 */
+	if (WARN_ON(master->smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC))
+		return ERR_PTR(-EOPNOTSUPP);
+
 	ret = iommu_copy_struct_from_user(&arg, user_data,
 					  IOMMU_HWPT_DATA_ARM_SMMUV3, ste);
 	if (ret)
diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
index 79afaef18906..6c8ae70c90fe 100644
--- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
+++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h
@@ -523,6 +523,7 @@ struct arm_smmu_cmdq_ent {
 		#define CMDQ_OP_TLBI_NH_ALL     0x10
 		#define CMDQ_OP_TLBI_NH_ASID	0x11
 		#define CMDQ_OP_TLBI_NH_VA	0x12
+		#define CMDQ_OP_TLBI_NH_VAA	0x13
 		#define CMDQ_OP_TLBI_EL2_ALL	0x20
 		#define CMDQ_OP_TLBI_EL2_ASID	0x21
 		#define CMDQ_OP_TLBI_EL2_VA	0x22
diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h
index fd7d16fd441d..f3aefb11f681 100644
--- a/include/uapi/linux/iommufd.h
+++ b/include/uapi/linux/iommufd.h
@@ -685,9 +685,11 @@ struct iommu_hwpt_get_dirty_bitmap {
  * enum iommu_hwpt_invalidate_data_type - IOMMU HWPT Cache Invalidation
  *                                        Data Type
  * @IOMMU_HWPT_INVALIDATE_DATA_VTD_S1: Invalidation data for VTD_S1
+ * @IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3: Invalidation data for ARM SMMUv3
  */
 enum iommu_hwpt_invalidate_data_type {
 	IOMMU_HWPT_INVALIDATE_DATA_VTD_S1 = 0,
+	IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3 = 1,
 };
 
 /**
@@ -726,6 +728,25 @@ struct iommu_hwpt_vtd_s1_invalidate {
 	__u32 __reserved;
 };
 
+/**
+ * struct iommu_hwpt_arm_smmuv3_invalidate - ARM SMMUv3 cahce invalidation
+ *         (IOMMU_HWPT_INVALIDATE_DATA_ARM_SMMUV3)
+ * @cmd: 128-bit cache invalidation command that runs in SMMU CMDQ.
+ *       Must be little-endian.
+ *
+ * Supported command list:
+ *     CMDQ_OP_TLBI_NSNH_ALL
+ *     CMDQ_OP_TLBI_NH_VA
+ *     CMDQ_OP_TLBI_NH_VAA
+ *     CMDQ_OP_TLBI_NH_ALL
+ *     CMDQ_OP_TLBI_NH_ASID
+ *
+ * -EIO will be returned if the command is not supported.
+ */
+struct iommu_hwpt_arm_smmuv3_invalidate {
+	__aligned_u64 cmd[2];
+};
+
 /**
  * struct iommu_hwpt_invalidate - ioctl(IOMMU_HWPT_INVALIDATE)
  * @size: sizeof(struct iommu_hwpt_invalidate)
-- 
2.43.0


  parent reply	other threads:[~2024-08-27 17:02 UTC|newest]

Thread overview: 149+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2024-08-27 16:59 [PATCH v2 00/19] iommufd: Add VIOMMU infrastructure (Part-1) Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 01/19] iommufd: Reorder struct forward declarations Nicolin Chen
2024-09-05 15:44   ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 02/19] iommufd/viommu: Add IOMMUFD_OBJ_VIOMMU and IOMMU_VIOMMU_ALLOC ioctl Nicolin Chen
2024-09-01  2:39   ` Baolu Lu
2024-09-02  5:27     ` Nicolin Chen
2024-09-04 16:26       ` Jason Gunthorpe
2024-09-04 17:29         ` Nicolin Chen
2024-09-04 23:37           ` Jason Gunthorpe
2024-09-05  3:07             ` Nicolin Chen
2024-09-12  3:39             ` Nicolin Chen
2024-09-15 21:10               ` Jason Gunthorpe
2024-09-05 15:53   ` Jason Gunthorpe
2024-09-05 17:10     ` Nicolin Chen
2024-09-05 17:41       ` Jason Gunthorpe
2024-09-05 18:04         ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 03/19] iommu: Pass in a viommu pointer to domain_alloc_user op Nicolin Chen
2024-09-05 15:54   ` Jason Gunthorpe
2024-09-05 17:19     ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 04/19] iommufd: Allow pt_id to carry viommu_id for IOMMU_HWPT_ALLOC Nicolin Chen
2024-09-05 15:56   ` Jason Gunthorpe
2024-09-26  8:50   ` Yi Liu
2024-09-26 20:10     ` Nicolin Chen
2024-09-27  0:43       ` Tian, Kevin
2024-09-27  1:25         ` Nicolin Chen
2024-09-27  2:23           ` Tian, Kevin
2024-09-27  2:57             ` Nicolin Chen
2024-09-27  5:38       ` Yi Liu
2024-09-27  6:02         ` Nicolin Chen
2024-09-27 11:59           ` Jason Gunthorpe
2024-09-27 19:52             ` Nicolin Chen
2024-09-27 12:12           ` Yi Liu
2024-09-27 20:29             ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 05/19] iommufd/selftest: Add IOMMU_VIOMMU_ALLOC test coverage Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 06/19] iommufd/viommu: Add IOMMU_VIOMMU_SET/UNSET_VDEV_ID ioctl Nicolin Chen
2024-09-05 16:03   ` Jason Gunthorpe
2024-09-05 17:37     ` Nicolin Chen
2024-09-05 17:43       ` Jason Gunthorpe
2024-09-05 20:14         ` Nicolin Chen
2024-09-11  6:19           ` Tian, Kevin
2024-09-11  7:11             ` Nicolin Chen
2024-09-11  7:18               ` Tian, Kevin
2024-09-11  7:47                 ` Nicolin Chen
2024-10-01  8:54       ` Nicolin Chen
2024-10-01 13:46         ` Jason Gunthorpe
2024-10-01 18:45           ` Nicolin Chen
2024-09-27 13:50   ` Mostafa Saleh
2024-09-27 14:01     ` Jason Gunthorpe
2024-09-27 14:22       ` Mostafa Saleh
2024-09-27 14:58         ` Jason Gunthorpe
2024-09-27 15:59           ` Mostafa Saleh
2024-10-04  4:32   ` Alexey Kardashevskiy
2024-10-04  5:33     ` Nicolin Chen
2024-10-04 11:41     ` Jason Gunthorpe
2024-10-04 18:13       ` Nicolin Chen
2024-10-04 18:50         ` Jason Gunthorpe
2024-10-04 19:25           ` Nicolin Chen
2024-10-04 20:17             ` Jason Gunthorpe
2024-10-04 20:33               ` Nicolin Chen
2024-10-07 17:18                 ` Jason Gunthorpe
2025-04-30  7:54       ` Alexey Kardashevskiy
2025-04-30  7:58         ` Nicolin Chen
2025-05-05 17:08           ` Jason Gunthorpe
2025-05-06  2:53             ` Nicolin Chen
2025-05-06  4:35               ` Alexey Kardashevskiy
2025-05-06  4:46               ` Tian, Kevin
2025-05-06 12:58               ` Jason Gunthorpe
2025-05-06 19:55                 ` Nicolin Chen
2025-05-07 12:28                   ` Jason Gunthorpe
2025-05-07 11:18                 ` Alexey Kardashevskiy
2025-05-07 12:24                   ` Jason Gunthorpe
2025-05-09  2:57                     ` Alexey Kardashevskiy
2025-05-09 22:07                       ` Jason Gunthorpe
2025-05-10 12:51                         ` Alexey Kardashevskiy
2025-05-12 15:31                           ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 07/19] iommufd/selftest: Add IOMMU_VIOMMU_SET/UNSET_VDEV_ID test coverage Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 08/19] iommufd/viommu: Add cache_invalidate for IOMMU_VIOMMU_TYPE_DEFAULT Nicolin Chen
2024-09-05 16:05   ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 09/19] iommufd: Allow hwpt_id to carry viommu_id for IOMMU_HWPT_INVALIDATE Nicolin Chen
2024-09-05 16:10   ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 10/19] iommufd/viommu: Add vdev_id helpers for IOMMU drivers Nicolin Chen
2024-09-05 16:14   ` Jason Gunthorpe
2024-09-05 17:53     ` Nicolin Chen
2024-09-11 23:11       ` Jason Gunthorpe
2024-09-12  3:17         ` Nicolin Chen
2024-10-05  5:19           ` Nicolin Chen
2024-10-07 15:38             ` Jason Gunthorpe
2024-10-07 16:36               ` Nicolin Chen
2024-10-07 17:11                 ` Jason Gunthorpe
2024-10-07 17:25                   ` Nicolin Chen
2024-10-07 18:28                     ` Jason Gunthorpe
2024-10-07 19:56                       ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 11/19] iommu: Add iommu_copy_struct_from_full_user_array helper Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 12/19] iommufd/selftest: Add mock_viommu_invalidate_user op Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 13/19] iommufd/selftest: Add IOMMU_TEST_OP_DEV_CHECK_CACHE test command Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 14/19] iommufd/selftest: Add VIOMMU coverage for IOMMU_HWPT_INVALIDATE ioctl Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 15/19] iommufd/viommu: Add iommufd_viommu_to_parent_domain helper Nicolin Chen
2024-08-27 16:59 ` Nicolin Chen [this message]
2024-09-05 16:23   ` [PATCH v2 16/19] iommu/arm-smmu-v3: Add arm_smmu_cache_invalidate_user Jason Gunthorpe
2024-09-05 17:56     ` Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 17/19] iommu/arm-smmu-v3: Add arm_smmu_viommu_cache_invalidate Nicolin Chen
2024-09-05 16:20   ` Jason Gunthorpe
2024-09-05 18:00     ` Nicolin Chen
2024-09-05 18:21       ` Jason Gunthorpe
2024-09-11  6:25         ` Tian, Kevin
2024-09-11  7:20           ` Nicolin Chen
2024-09-11  7:50             ` Baolu Lu
2024-09-11  8:17               ` Tian, Kevin
2024-09-11  8:19                 ` Baolu Lu
2024-09-11 21:08                 ` Nicolin Chen
2024-09-12  4:45                   ` Baolu Lu
2024-09-11  8:13             ` Tian, Kevin
2024-09-11 20:53               ` Nicolin Chen
2024-09-11 23:07               ` Jason Gunthorpe
2024-09-13  2:33                 ` Tian, Kevin
2024-09-14 14:50                   ` Jason Gunthorpe
2024-09-18  8:10                     ` Tian, Kevin
2024-09-23 18:34                       ` Jason Gunthorpe
2024-09-11 23:00           ` Jason Gunthorpe
2024-08-27 16:59 ` [PATCH v2 18/19] iommu/arm-smmu-v3: Allow ATS for IOMMU_DOMAIN_NESTED Nicolin Chen
2024-08-27 16:59 ` [PATCH v2 19/19] iommu/arm-smmu-v3: Update comments about ATS and bypass Nicolin Chen
2024-09-11  6:12 ` [PATCH v2 00/19] iommufd: Add VIOMMU infrastructure (Part-1) Tian, Kevin
2024-09-11  7:08   ` Nicolin Chen
2024-09-11  7:18     ` Tian, Kevin
2024-09-11  7:40       ` Nicolin Chen
2024-09-11  8:08         ` Tian, Kevin
2024-09-11 20:21           ` Nicolin Chen
2024-10-01  1:55     ` Alexey Kardashevskiy
2024-10-01  3:36       ` Nicolin Chen
2024-10-01  5:06         ` Alexey Kardashevskiy
2024-10-01 13:44           ` Jason Gunthorpe
2024-09-25 10:30 ` Yi Liu
2024-09-25 18:55   ` Nicolin Chen
2024-09-26  8:47     ` Yi Liu
2024-09-26 20:03       ` Nicolin Chen
2024-09-27  2:05         ` Baolu Lu
2024-09-27  6:14           ` Yi Liu
2024-09-27  5:54         ` Yi Liu
2024-09-27  6:32           ` Nicolin Chen
2024-09-27 12:12             ` Yi Liu
2024-09-27 12:20               ` Jason Gunthorpe
2024-09-29  7:19                 ` Yi Liu
2024-10-01 13:48                   ` Jason Gunthorpe
2024-10-01 18:40                     ` Nicolin Chen
2024-09-27 20:44               ` Nicolin Chen
2024-09-29  7:16                 ` Yi Liu
2024-09-30 21:59                   ` Nicolin Chen
2024-10-09  7:20                     ` Yi Liu
2024-10-09 12:31                       ` Jason Gunthorpe

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=3548bfff43a0e1c072b77fc37a1a8a6c930bcec9.1724776335.git.nicolinc@nvidia.com \
    --to=nicolinc@nvidia.com \
    --cc=baolu.lu@linux.intel.com \
    --cc=dwmw2@infradead.org \
    --cc=eric.auger@redhat.com \
    --cc=iommu@lists.linux.dev \
    --cc=jean-philippe@linaro.org \
    --cc=jgg@nvidia.com \
    --cc=joro@8bytes.org \
    --cc=kevin.tian@intel.com \
    --cc=linux-arm-kernel@lists.infradead.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-kselftest@vger.kernel.org \
    --cc=mdf@kernel.org \
    --cc=mshavit@google.com \
    --cc=robin.murphy@arm.com \
    --cc=shameerali.kolothum.thodi@huawei.com \
    --cc=shuah@kernel.org \
    --cc=smostafa@google.com \
    --cc=suravee.suthikulpanit@amd.com \
    --cc=will@kernel.org \
    --cc=yi.l.liu@intel.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®