mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Andrew Jones <andrew.jones@oss.qualcomm.com>
To: linux-riscv@lists.infradead.org, iommu@lists.linux.dev
Cc: linux-kernel@vger.kernel.org, tomasz.jeznach@linux.dev,
	tjeznach@rivosinc.com, jgg@ziepe.ca, jgg@nvidia.com,
	joro@8bytes.org, will@kernel.org, robin.murphy@arm.com,
	pjw@kernel.org, palmer@dabbelt.com, anup@brainfault.org,
	tglx@kernel.org, kevin.tian@intel.com,
	fangyu.yu@linux.alibaba.com
Subject: [PATCH v4 04/21] iommu/dma: Add iommu_dma_map_msi()
Date: Thu, 20 Aug 2026 23:41:33 +0200	[thread overview]
Message-ID: <20260820214150.545737-5-andrew.jones@oss.qualcomm.com> (raw)
In-Reply-To: <20260820214150.545737-1-andrew.jones@oss.qualcomm.com>

Interrupt-remapping drivers may need to map MSI targets before any MSI
descriptor exists and retain their own PA-to-IOVA lookup table. Add
iommu_dma_map_msi() to return the IOVA and mapping granule for both
DMA-IOMMU and iommufd domains.

Allow callers to require an exact mapping size. This prevents doorbells
such as RISC-V IMSIC files from sharing a larger IOMMU leaf.

Building a table requires the domain to remain stable across every
mapping. The IOMMU group mutex provides that serialization, but its
structure is private to the core. Expose lock helpers and a scoped guard
so external callers can hold it for the complete operation.

Signed-off-by: Andrew Jones <andrew.jones@oss.qualcomm.com>
---
 drivers/iommu/dma-iommu.c      |  22 +------
 drivers/iommu/dma-iommu.h      |   8 ---
 drivers/iommu/iommu-priv.h     |   8 ---
 drivers/iommu/iommu.c          | 104 ++++++++++++++++++++++++++++-----
 drivers/iommu/iommufd/driver.c |  31 ++--------
 include/linux/iommu.h          |  31 ++++++++++
 6 files changed, 127 insertions(+), 77 deletions(-)

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 3ce9f4878d92..39ee5f0b1da9 100644
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -2248,9 +2248,9 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
 }
 
 /*
- * Descriptor-free counterpart to iommu_dma_sw_msi(). Maps an MSI physical
- * page into the domain and returns the IOVA and mapping granule. Used for
- * pre-mapping MSI targets before any MSI descriptor has been set.
+ * Maps an MSI physical page into the domain and returns the IOVA and
+ * mapping granule. Used for pre-mapping MSI targets before any MSI
+ * descriptor has been set.
  *
  * The caller must pass a device attached to @domain and hold @dev's IOMMU
  * group mutex. If @required_size is non-zero then it must exactly match the
@@ -2284,22 +2284,6 @@ int iommu_dma_sw_map_msi(struct iommu_domain *domain,
 	return 0;
 }
 
-int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
-		     phys_addr_t msi_addr)
-{
-	struct device *dev = msi_desc_to_dev(desc);
-	dma_addr_t msi_iova;
-	unsigned int msi_shift;
-	int ret;
-
-	ret = iommu_dma_sw_map_msi(domain, dev, msi_addr, 0, &msi_iova, &msi_shift);
-	if (ret)
-		return ret;
-
-	msi_desc_set_iommu_msi_iova(desc, msi_iova, msi_shift);
-	return 0;
-}
-
 static int iommu_dma_init(void)
 {
 	if (is_kdump_kernel())
diff --git a/drivers/iommu/dma-iommu.h b/drivers/iommu/dma-iommu.h
index 3e1adeafdcac..7d707157aebb 100644
--- a/drivers/iommu/dma-iommu.h
+++ b/drivers/iommu/dma-iommu.h
@@ -23,8 +23,6 @@ int iommu_dma_sw_map_msi(struct iommu_domain *domain,
 			 struct device *dev, phys_addr_t msi_addr,
 			 size_t required_size, dma_addr_t *msi_iova,
 			 unsigned int *msi_shift);
-int iommu_dma_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
-		     phys_addr_t msi_addr);
 
 extern bool iommu_dma_forcedac;
 
@@ -65,11 +63,5 @@ static inline int iommu_dma_sw_map_msi(struct iommu_domain *domain,
 	return -ENODEV;
 }
 
-static inline int iommu_dma_sw_msi(struct iommu_domain *domain,
-				   struct msi_desc *desc, phys_addr_t msi_addr)
-{
-	return -ENODEV;
-}
-
 #endif	/* CONFIG_IOMMU_DMA */
 #endif	/* __DMA_IOMMU_H */
diff --git a/drivers/iommu/iommu-priv.h b/drivers/iommu/iommu-priv.h
index f60373cd2f70..109aca07470b 100644
--- a/drivers/iommu/iommu-priv.h
+++ b/drivers/iommu/iommu-priv.h
@@ -53,18 +53,10 @@ int iommu_replace_group_handle(struct iommu_group *group,
 			       struct iommu_attach_handle *handle);
 
 #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE) && IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
-int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
-		   phys_addr_t msi_addr);
 int iommufd_sw_map_msi(struct iommu_domain *domain, struct device *dev,
 		       phys_addr_t msi_addr, size_t required_size,
 		       dma_addr_t *msi_iova, unsigned int *msi_shift);
 #else /* !CONFIG_IOMMUFD_DRIVER_CORE || !CONFIG_IRQ_MSI_IOMMU */
-static inline int iommufd_sw_msi(struct iommu_domain *domain,
-				 struct msi_desc *desc, phys_addr_t msi_addr)
-{
-	return -EOPNOTSUPP;
-}
-
 static inline int iommufd_sw_map_msi(struct iommu_domain *domain,
 				     struct device *dev, phys_addr_t msi_addr,
 				     size_t required_size, dma_addr_t *msi_iova,
diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index a3dea7353500..84b60d0f8c23 100644
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -1364,15 +1364,46 @@ void iommu_group_remove_device(struct device *dev)
 }
 EXPORT_SYMBOL_GPL(iommu_group_remove_device);
 
-#if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
+#if IS_ENABLED(CONFIG_IOMMU_API)
+/*
+ * iommu_group_mutex_lock(), iommu_group_mutex_unlock(), and
+ * iommu_group_mutex_assert() must be called after device group param is
+ * set.
+ */
+
+/**
+ * iommu_group_mutex_lock - Lock the iommu group mutex for a device
+ * @dev: the device whose group mutex should be locked
+ *
+ * Callers that need to invoke a function documented as requiring the
+ * device's iommu group mutex (e.g. iommu_dma_map_msi()) from outside
+ * drivers/iommu/ use this instead of reaching into struct iommu_group,
+ * which is private to the core. Must be paired with
+ * iommu_group_mutex_unlock().
+ */
+void iommu_group_mutex_lock(struct device *dev)
+{
+	mutex_lock(&dev->iommu_group->mutex);
+}
+EXPORT_SYMBOL_GPL(iommu_group_mutex_lock);
+
+/**
+ * iommu_group_mutex_unlock - Unlock the iommu group mutex for a device
+ * @dev: the device whose group mutex should be unlocked
+ */
+void iommu_group_mutex_unlock(struct device *dev)
+{
+	mutex_unlock(&dev->iommu_group->mutex);
+}
+EXPORT_SYMBOL_GPL(iommu_group_mutex_unlock);
+
+#if IS_ENABLED(CONFIG_LOCKDEP)
 /**
  * iommu_group_mutex_assert - Check device group mutex lock
  * @dev: the device that has group param set
  *
  * This function is called by an iommu driver to check whether it holds
  * group mutex lock for the given device or not.
- *
- * Note that this function must be called after device group param is set.
  */
 void iommu_group_mutex_assert(struct device *dev)
 {
@@ -1381,7 +1412,8 @@ void iommu_group_mutex_assert(struct device *dev)
 	lockdep_assert_held(&group->mutex);
 }
 EXPORT_SYMBOL_GPL(iommu_group_mutex_assert);
-#endif
+#endif /* CONFIG_LOCKDEP */
+#endif /* CONFIG_IOMMU_API */
 
 static struct device *iommu_group_first_dev(struct iommu_group *group)
 {
@@ -4223,6 +4255,52 @@ void pci_dev_reset_iommu_done(struct pci_dev *pdev)
 EXPORT_SYMBOL_GPL(pci_dev_reset_iommu_done);
 
 #if IS_ENABLED(CONFIG_IRQ_MSI_IOMMU)
+/**
+ * iommu_dma_map_msi() - Map an MSI page in an IOMMU domain
+ * @domain: IOMMU domain to map into
+ * @dev: Device used to allocate the IOVA
+ * @msi_addr: MSI target address to be mapped
+ * @required_size: Required mapping size, or 0 to accept any size
+ * @msi_iova: IOVA for @msi_addr, or 0 for passthrough
+ * @msi_shift: Mapping granule shift, or 0 for passthrough
+ *
+ * The caller must hold @dev's iommu group mutex, e.g. via
+ * iommu_group_mutex_lock()/iommu_group_mutex_unlock(). This function does
+ * not take the mutex itself because callers building a table of mappings
+ * (e.g. one IOVA per possible CPU's IMSIC page) call it in a loop; locking
+ * inside would mean re-acquiring the mutex on every iteration and would not
+ * stop the domain from changing between iterations, leaving the table
+ * inconsistent. The caller locks once around the whole loop instead.
+ *
+ * Return: 0 on success or negative error code if the mapping failed.
+ */
+int iommu_dma_map_msi(struct iommu_domain *domain,
+		      struct device *dev, phys_addr_t msi_addr,
+		      size_t required_size, dma_addr_t *msi_iova,
+		      unsigned int *msi_shift)
+{
+	*msi_iova = 0;
+	*msi_shift = 0;
+
+	if (!domain)
+		return -EINVAL;
+
+	if (domain->type == IOMMU_DOMAIN_IDENTITY)
+		return 0;
+
+	switch (domain->cookie_type) {
+	case IOMMU_COOKIE_DMA_MSI:
+	case IOMMU_COOKIE_DMA_IOVA:
+		return iommu_dma_sw_map_msi(domain, dev, msi_addr,
+					    required_size, msi_iova, msi_shift);
+	case IOMMU_COOKIE_IOMMUFD:
+		return iommufd_sw_map_msi(domain, dev, msi_addr,
+					  required_size, msi_iova, msi_shift);
+	default:
+		return -EOPNOTSUPP;
+	}
+}
+
 /**
  * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
  * @desc: MSI descriptor, will store the MSI page
@@ -4238,6 +4316,8 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 {
 	struct device *dev = msi_desc_to_dev(desc);
 	struct iommu_group *group = dev->iommu_group;
+	dma_addr_t msi_iova;
+	unsigned int msi_shift;
 	int ret = 0;
 
 	if (!group)
@@ -4246,18 +4326,10 @@ int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
 	mutex_lock(&group->mutex);
 	/* An IDENTITY domain must pass through */
 	if (group->domain && group->domain->type != IOMMU_DOMAIN_IDENTITY) {
-		switch (group->domain->cookie_type) {
-		case IOMMU_COOKIE_DMA_MSI:
-		case IOMMU_COOKIE_DMA_IOVA:
-			ret = iommu_dma_sw_msi(group->domain, desc, msi_addr);
-			break;
-		case IOMMU_COOKIE_IOMMUFD:
-			ret = iommufd_sw_msi(group->domain, desc, msi_addr);
-			break;
-		default:
-			ret = -EOPNOTSUPP;
-			break;
-		}
+		ret = iommu_dma_map_msi(group->domain, dev, msi_addr, 0,
+					&msi_iova, &msi_shift);
+		if (!ret)
+			msi_desc_set_iommu_msi_iova(desc, msi_iova, msi_shift);
 	}
 	mutex_unlock(&group->mutex);
 	return ret;
diff --git a/drivers/iommu/iommufd/driver.c b/drivers/iommu/iommufd/driver.c
index 69b3dbcbee3b..b39796bc7251 100644
--- a/drivers/iommu/iommufd/driver.c
+++ b/drivers/iommu/iommufd/driver.c
@@ -249,11 +249,11 @@ int iommufd_sw_msi_install(struct iommufd_ctx *ictx,
 EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi_install, "IOMMUFD_INTERNAL");
 
 /*
- * Descriptor-free counterpart to iommufd_sw_msi(). Maps an MSI physical page
- * into the domain and returns the IOVA. Used for pre-mapping MSI targets before
- * any MSI descriptor has been set (e.g. IMSIC doorbell pages). The IOVA is
- * global to the iommufd file descriptor: every domain and device using the
- * same MSI parameters gets the same IOVA.
+ * Maps an MSI physical page into the domain and returns the IOVA. Used for
+ * pre-mapping MSI targets before any MSI descriptor has been set (e.g.
+ * IMSIC doorbell pages). The IOVA is global to the iommufd file
+ * descriptor: every domain and device using the same MSI parameters gets
+ * the same IOVA.
  *
  * msi_addr is the exact byte offset of the MSI doorbell; the caller must have
  * verified it is contained within an MMIO region safe to map at PAGE_SIZE. If
@@ -321,27 +321,6 @@ int iommufd_sw_map_msi(struct iommu_domain *domain, struct device *dev,
 	return 0;
 }
 EXPORT_SYMBOL_NS_GPL(iommufd_sw_map_msi, "IOMMUFD");
-
-/*
- * Called by the irq layer when the platform translates MSI addresses through
- * the IOMMU. Wraps iommufd_sw_map_msi() and stores the result in the descriptor.
- */
-int iommufd_sw_msi(struct iommu_domain *domain, struct msi_desc *desc,
-		   phys_addr_t msi_addr)
-{
-	dma_addr_t msi_iova;
-	unsigned int msi_shift;
-	int rc;
-
-	rc = iommufd_sw_map_msi(domain, msi_desc_to_dev(desc), msi_addr,
-				0, &msi_iova, &msi_shift);
-	if (rc)
-		return rc;
-
-	msi_desc_set_iommu_msi_iova(desc, msi_iova, msi_shift);
-	return 0;
-}
-EXPORT_SYMBOL_NS_GPL(iommufd_sw_msi, "IOMMUFD");
 #endif
 
 MODULE_DESCRIPTION("iommufd code shared with builtin modules");
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index ac43b8b93f14..868b14b63e77 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -8,6 +8,7 @@
 #define __LINUX_IOMMU_H
 
 #include <linux/scatterlist.h>
+#include <linux/cleanup.h>
 #include <linux/device.h>
 #include <linux/types.h>
 #include <linux/errno.h>
@@ -1561,8 +1562,22 @@ static inline void pci_dev_reset_iommu_done(struct pci_dev *pdev)
 
 #ifdef CONFIG_IRQ_MSI_IOMMU
 #ifdef CONFIG_IOMMU_API
+int iommu_dma_map_msi(struct iommu_domain *domain,
+		      struct device *dev, phys_addr_t msi_addr,
+		      size_t required_size, dma_addr_t *msi_iova,
+		      unsigned int *msi_shift);
 int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr);
 #else
+static inline int iommu_dma_map_msi(struct iommu_domain *domain,
+				    struct device *dev, phys_addr_t msi_addr,
+				    size_t required_size, dma_addr_t *msi_iova,
+				    unsigned int *msi_shift)
+{
+	*msi_iova = 0;
+	*msi_shift = 0;
+	return 0;
+}
+
 static inline int iommu_dma_prepare_msi(struct msi_desc *desc,
 					phys_addr_t msi_addr)
 {
@@ -1571,6 +1586,22 @@ static inline int iommu_dma_prepare_msi(struct msi_desc *desc,
 #endif /* CONFIG_IOMMU_API */
 #endif /* CONFIG_IRQ_MSI_IOMMU */
 
+#if IS_ENABLED(CONFIG_IOMMU_API)
+void iommu_group_mutex_lock(struct device *dev);
+void iommu_group_mutex_unlock(struct device *dev);
+#else
+static inline void iommu_group_mutex_lock(struct device *dev) { }
+static inline void iommu_group_mutex_unlock(struct device *dev) { }
+#endif
+
+/*
+ * scoped_guard(iommu_group, dev) { ... } locks dev's iommu group mutex for
+ * the scope of the block. See iommu_group_mutex_lock().
+ */
+DEFINE_LOCK_GUARD_1(iommu_group, struct device,
+		    iommu_group_mutex_lock(_T->lock),
+		    iommu_group_mutex_unlock(_T->lock))
+
 #if IS_ENABLED(CONFIG_LOCKDEP) && IS_ENABLED(CONFIG_IOMMU_API)
 void iommu_group_mutex_assert(struct device *dev);
 #else
-- 
2.43.0


  parent reply	other threads:[~2026-08-20 21:42 UTC|newest]

Thread overview: 57+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-08-20 21:41 [PATCH v4 00/21] iommu/riscv: Enable MSI remapping, IOMMU_DMA and VFIO Andrew Jones
2026-08-20 21:41 ` [PATCH v4 01/21] iommufd: Convert struct iommufd_sw_msi_maps to a growable bitmap Andrew Jones
2026-08-20 21:41 ` [PATCH v4 02/21] iommufd: Add iommufd_sw_map_msi() Andrew Jones
2026-08-20 22:09   ` Jason Gunthorpe
2026-08-21 11:07     ` Andrew Jones
2026-08-21 12:02       ` Jason Gunthorpe
2026-08-21 13:47         ` Andrew Jones
2026-08-21 14:00           ` Jason Gunthorpe
2026-08-21 14:23             ` Andrew Jones
2026-08-21 14:31               ` Jason Gunthorpe
2026-08-21 15:18                 ` Andrew Jones
2026-08-21 16:12                   ` Jason Gunthorpe
2026-08-21 17:12                     ` Andrew Jones
2026-08-21 17:20                       ` Jason Gunthorpe
2026-08-22 13:50                         ` Andrew Jones
2026-08-22 14:06                           ` Jason Gunthorpe
2026-08-24  8:50                             ` Andrew Jones
2026-08-24 12:52                               ` Jason Gunthorpe
2026-08-24 15:43                                 ` Andrew Jones
2026-08-24 17:04                                   ` Jason Gunthorpe
2026-08-25 13:24                                     ` Andrew Jones
2026-08-25 14:03                                       ` Jason Gunthorpe
2026-08-25 16:14                                         ` Andrew Jones
2026-08-25 16:52                                           ` Jason Gunthorpe
2026-08-25 17:38                                             ` Andrew Jones
2026-08-25 17:59                                               ` Jason Gunthorpe
2026-08-26  8:23                                                 ` Andrew Jones
2026-08-26 14:48                                                   ` Andrew Jones
2026-08-26 16:32                                                   ` Jason Gunthorpe
2026-08-26 18:11                                                     ` Andrew Jones
2026-08-27  8:47                                                       ` Andrew Jones
2026-08-27 12:11                                                         ` Jason Gunthorpe
2026-08-20 21:41 ` [PATCH v4 03/21] iommu/dma: Add iommu_dma_sw_map_msi() Andrew Jones
2026-08-20 21:41 ` Andrew Jones [this message]
2026-08-20 21:41 ` [PATCH v4 05/21] iommu: Document MSI mapping during domain replacement Andrew Jones
2026-08-20 21:41 ` [PATCH v4 06/21] genirq/msi: Provide DOMAIN_BUS_MSI_REMAP Andrew Jones
2026-08-20 21:41 ` [PATCH v4 07/21] irqchip/riscv-imsic: Compose MSI updates through the hierarchy Andrew Jones
2026-08-20 21:41 ` [PATCH v4 08/21] iommu/riscv: Add IRQ domain for interrupt remapping Andrew Jones
2026-08-20 21:41 ` [PATCH v4 09/21] iommu/riscv: Refresh platform MSI domain before IR setup Andrew Jones
2026-08-20 21:41 ` [PATCH v4 10/21] iommu/riscv: Prepare info->domain for concurrent RCU read access Andrew Jones
2026-08-20 21:41 ` [PATCH v4 11/21] iommu/riscv: Reserve an MSI IOVA window for iommufd Andrew Jones
2026-08-20 21:41 ` [PATCH v4 12/21] iommu/riscv: Pre-map IMSIC MSI targets Andrew Jones
2026-08-20 21:41 ` [PATCH v4 13/21] iommu/riscv: Preserve MSI IOVA state across domain replacement Andrew Jones
2026-08-20 21:52   ` Jason Gunthorpe
2026-08-21 11:14     ` Andrew Jones
2026-08-21 13:22       ` Jason Gunthorpe
2026-08-21 13:56         ` Andrew Jones
2026-08-20 21:41 ` [PATCH v4 14/21] iommu/riscv: Gate direct identity boundary switches with live MSIs Andrew Jones
2026-08-20 21:41 ` [PATCH v4 15/21] iommu/riscv: Remap IMSIC targets during MSI composition Andrew Jones
2026-08-20 21:41 ` [PATCH v4 16/21] iommu/dma: Enable IOMMU_DMA for 64-bit RISC-V Andrew Jones
2026-08-20 21:41 ` [PATCH v4 17/21] iommu/riscv: Report cache coherency capability Andrew Jones
2026-08-20 21:47   ` Jason Gunthorpe
2026-08-21 11:15     ` Andrew Jones
2026-08-20 21:41 ` [PATCH v4 18/21] vfio: enable IOMMU_TYPE1 for RISC-V Andrew Jones
2026-08-20 21:41 ` [PATCH v4 19/21] RISC-V: KVM: Enable KVM_VFIO interfaces on RISC-V arch Andrew Jones
2026-08-20 21:41 ` [PATCH v4 20/21] riscv: defconfig: Enable IOMMUFD and VFIO Andrew Jones
2026-08-20 21:41 ` [PATCH v4 21/21] selftests/vfio: Allow building on RISC-V Andrew Jones

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20260820214150.545737-5-andrew.jones@oss.qualcomm.com \
    --to=andrew.jones@oss.qualcomm.com \
    --cc=anup@brainfault.org \
    --cc=fangyu.yu@linux.alibaba.com \
    --cc=iommu@lists.linux.dev \
    --cc=jgg@nvidia.com \
    --cc=jgg@ziepe.ca \
    --cc=joro@8bytes.org \
    --cc=kevin.tian@intel.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-riscv@lists.infradead.org \
    --cc=palmer@dabbelt.com \
    --cc=pjw@kernel.org \
    --cc=robin.murphy@arm.com \
    --cc=tglx@kernel.org \
    --cc=tjeznach@rivosinc.com \
    --cc=tomasz.jeznach@linux.dev \
    --cc=will@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®