[PATCH 11/14] rust: pci: add C FFI support to typed SR-IOV PF registration data

[Zhi Wang <zhiw@nvidia.com>]

Rust VF drivers can borrow PF-owned registration data directly, but C VF
drivers need an ABI-checked operations table and a pinned context.

Place an optional struct rust_ffi descriptor at offset zero in the
registration header while retaining the Rust TypeId. Initialize
registrations created with new() with an empty descriptor, and add
new_ffi() to construct a descriptor for the same pinned payload.

Add pci_iov_borrow_rust_pf_data() to locate the PF descriptor and validate
the token, ABI version, and operations-table size. Require managed SR-IOV
so device-link ordering and registration teardown keep the descriptor and
its context alive until the VF driver is unbound.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 drivers/pci/iov.c          |  51 +++++++++++++++++
 include/linux/pci.h        |  34 +++++++++++-
 rust/kernel/interop/ffi.rs |  15 ++++-
 rust/kernel/pci/sriov.rs   | 110 ++++++++++++++++++++++++++++---------
 4 files changed, 179 insertions(+), 31 deletions(-)

diff --git a/drivers/pci/iov.c b/drivers/pci/iov.c
index ee5eff209e15..0f3ae75b8a7c 100644
--- a/drivers/pci/iov.c
+++ b/drivers/pci/iov.c
@@ -11,6 +11,7 @@
 #include <linux/bits.h>
 #include <linux/log2.h>
 #include <linux/pci.h>
+#include <linux/rust_ffi.h>
 #include <linux/sizes.h>
 #include <linux/slab.h>
 #include <linux/export.h>
@@ -80,6 +81,56 @@ void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver)
 }
 EXPORT_SYMBOL_GPL(pci_iov_get_pf_drvdata);
 
+#ifdef CONFIG_RUST
+/**
+ * pci_iov_borrow_rust_pf_data - Validate and borrow Rust data from a VF's PF
+ * @dev: VF PCI device
+ * @token: Required FFI ABI token
+ * @abi_major: Required ABI major version
+ * @min_abi_minor: Minimum required ABI minor version
+ * @required_ops_size: Minimum required size of the operations table
+ *
+ * This may be called from a VF driver's probe() callback or from a context in
+ * which the VF driver is known to remain attached. If probe() succeeds, the
+ * returned pointer, its operations table, and its context are borrowed until
+ * the VF driver is fully unbound, including the return of its remove() callback
+ * when present. If probe() fails, the caller must discard the borrow before
+ * returning. The caller must drain all work that can use the FFI before the
+ * borrow ends.
+ *
+ * The PF must publish an immutable descriptor before enabling VFs, use
+ * managed_sriov, and retain the descriptor until its remove() callback.
+ * Managed SR-IOV installs a device link from every VF to its PF before the VF
+ * can probe. The driver core therefore waits for an in-progress VF probe and
+ * unbinds a bound VF before invoking the PF driver's remove() callback.
+ *
+ * Return: A borrowed FFI descriptor, or an ERR_PTR() value on failure.
+ */
+const struct rust_ffi *
+pci_iov_borrow_rust_pf_data(struct pci_dev *dev,
+			    const struct rust_ffi_token *token,
+			    u16 abi_major, u16 min_abi_minor,
+			    size_t required_ops_size)
+{
+	const struct rust_ffi *ffi;
+	struct pci_dev *pf_dev;
+	struct pci_driver *pf_driver;
+
+	if (!dev->is_virtfn)
+		return ERR_PTR(-EINVAL);
+
+	pf_dev = pci_physfn(dev);
+	pf_driver = READ_ONCE(pf_dev->driver);
+	if (!pf_driver || !READ_ONCE(pf_driver->managed_sriov))
+		return ERR_PTR(-ENODEV);
+
+	ffi = READ_ONCE(pf_dev->vf_registration_data_rust);
+	return rust_ffi_borrow(ffi, token, abi_major, min_abi_minor,
+			       required_ops_size);
+}
+EXPORT_SYMBOL_GPL(pci_iov_borrow_rust_pf_data);
+#endif
+
 /*
  * Per SR-IOV spec sec 3.3.10 and 3.3.11, First VF Offset and VF Stride may
  * change when NumVFs changes.
diff --git a/include/linux/pci.h b/include/linux/pci.h
index 1ccc7fee7495..4a7189ee247f 100644
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -339,6 +339,8 @@ struct pcie_link_state;
 struct pci_sriov;
 struct pci_p2pdma;
 struct rcec_ea;
+struct rust_ffi;
+struct rust_ffi_token;
 
 /* struct pci_dev - describes a PCI device
  *
@@ -352,9 +354,10 @@ struct rcec_ea;
  *			Such bridges are allocated additional MMIO and bus
  *			number resources to allow for hierarchy expansion.
  * @is_pciehp:		PCIe Hot-Plug Capable bridge.
- * @vf_registration_data_rust: Rust registration data published by the PF
- *			before enabling VFs and retained until all VFs are
- *			removed. The PF driver must use managed_sriov.
+ * @vf_registration_data_rust: Rust registration data beginning with a
+ *			struct rust_ffi, published by the PF before
+ *			enabling VFs and retained until all VFs are removed.
+ *			The PF driver must use managed_sriov.
  */
 struct pci_dev {
 	struct list_head bus_list;	/* Node in per-bus list */
@@ -2614,6 +2617,22 @@ int pci_iov_virtfn_bus(struct pci_dev *dev, int id);
 int pci_iov_virtfn_devfn(struct pci_dev *dev, int id);
 int pci_iov_vf_id(struct pci_dev *dev);
 void *pci_iov_get_pf_drvdata(struct pci_dev *dev, struct pci_driver *pf_driver);
+#ifdef CONFIG_RUST
+const struct rust_ffi *
+pci_iov_borrow_rust_pf_data(struct pci_dev *dev,
+			    const struct rust_ffi_token *token,
+			    u16 abi_major, u16 min_abi_minor,
+			    size_t required_ops_size);
+#else
+static inline const struct rust_ffi *
+pci_iov_borrow_rust_pf_data(struct pci_dev *dev,
+			    const struct rust_ffi_token *token,
+			    u16 abi_major, u16 min_abi_minor,
+			    size_t required_ops_size)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+#endif
 int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn);
 void pci_disable_sriov(struct pci_dev *dev);
 
@@ -2656,6 +2675,15 @@ static inline void *pci_iov_get_pf_drvdata(struct pci_dev *dev,
 	return ERR_PTR(-EINVAL);
 }
 
+static inline const struct rust_ffi *
+pci_iov_borrow_rust_pf_data(struct pci_dev *dev,
+			    const struct rust_ffi_token *token,
+			    u16 abi_major, u16 min_abi_minor,
+			    size_t required_ops_size)
+{
+	return ERR_PTR(-EOPNOTSUPP);
+}
+
 static inline int pci_enable_sriov(struct pci_dev *dev, int nr_virtfn)
 { return -ENODEV; }
 
diff --git a/rust/kernel/interop/ffi.rs b/rust/kernel/interop/ffi.rs
index a8c16a29110a..ea23bfecad32 100644
--- a/rust/kernel/interop/ffi.rs
+++ b/rust/kernel/interop/ffi.rs
@@ -99,10 +99,23 @@ pub unsafe trait Abi: 'static {
 /// [`struct rust_ffi`](srctree/include/linux/rust_ffi.h). It neither owns nor borrows the
 /// operations table or Rust context at the type level. The transport that publishes it must ensure
 /// that `ops` remains valid and that `context` remains alive at a stable address until all
-/// consumers have stopped using the descriptor.
+/// consumers have stopped using the descriptor. The default value exposes no operations.
 #[repr(transparent)]
 pub struct Descriptor(bindings::rust_ffi);
 
+impl Default for Descriptor {
+    fn default() -> Self {
+        Self(bindings::rust_ffi {
+            token: bindings::rust_ffi_token { high: 0, low: 0 },
+            abi_major: 0,
+            abi_minor: 0,
+            ops_size: 0,
+            ops: core::ptr::null(),
+            context: core::ptr::null(),
+        })
+    }
+}
+
 impl Descriptor {
     /// Creates a descriptor for a pinned Rust context.
     ///
diff --git a/rust/kernel/pci/sriov.rs b/rust/kernel/pci/sriov.rs
index efbe444e0733..a2590b66f78b 100644
--- a/rust/kernel/pci/sriov.rs
+++ b/rust/kernel/pci/sriov.rs
@@ -6,6 +6,10 @@
 use crate::{
     bindings,
     device, //
+    interop::ffi::{
+        Abi,
+        Descriptor, //
+    },
     prelude::*,
     types::{
         CovariantForLt,
@@ -91,26 +95,45 @@ pub fn num_vfs(&self) -> i32 {
 // depends on its device context.
 kernel::impl_device_context_deref!(unsafe { Device });
 
+#[repr(C)]
+struct VfRegistrationHeader {
+    ffi: Descriptor,
+    type_id: TypeId,
+}
+
+static_assert!(core::mem::offset_of!(VfRegistrationHeader, ffi) == 0);
+
 #[repr(C)]
 #[pin_data]
 struct VfRegistrationData<'a, F: ForLt + 'static> {
-    type_id: TypeId,
+    header: VfRegistrationHeader,
     #[pin]
     data: F::Of<'a>,
 }
 
 static_assert!(
-    core::mem::offset_of!(VfRegistrationData<'static, CovariantForLt!(())>, type_id) == 0
+    core::mem::offset_of!(VfRegistrationData<'static, CovariantForLt!(())>, header) == 0
 );
 
 impl<'a, F: ForLt + 'static> VfRegistrationData<'a, F> {
-    fn new<D>(data: D) -> impl PinInit<Self, Error> + use<'a, D, F>
+    fn new<D, M>(data: D, make_descriptor: M) -> impl PinInit<Self, Error> + use<'a, D, F, M>
     where
         D: PinInit<F::Of<'a>, Error> + 'a,
+        M: FnOnce(Pin<&F::Of<'a>>) -> Descriptor + 'a,
     {
-        try_pin_init!(Self {
-            type_id: TypeId::of::<F>(),
+        try_pin_init!(&this in Self {
+            header: VfRegistrationHeader {
+                ffi: Descriptor::default(),
+                type_id: TypeId::of::<F>(),
+            },
             data <- data,
+            _: {
+                // SAFETY: `data` has been initialized in place and will remain pinned at this
+                // address.
+                let data = unsafe { Pin::new_unchecked(&(*this.as_ptr()).data) };
+                // SAFETY: `header.ffi` is initialized and exclusively owned during construction.
+                unsafe { (*this.as_ptr()).header.ffi = make_descriptor(data) };
+            },
         })
     }
 }
@@ -138,28 +161,14 @@ impl<'a, F: ForLt + 'static> VfRegistration<'a, F>
 where
     for<'b> F::Of<'b>: Send + Sync,
 {
-    /// Publishes typed PF data for bound VF drivers.
-    ///
-    /// This returns a pin-initializer so the registration and payload can be embedded directly in
-    /// the PF driver's pinned data.
-    ///
-    /// Initialization returns [`ENODEV`] for a VF and [`EBUSY`] if the PF has enabled VFs or
-    /// already has a registration.
-    ///
-    /// # Safety
-    ///
-    /// The caller must invoke this during the PCI driver's probe and embed the result in the driver
-    /// data. On an SR-IOV PF, no VF may be enabled before probe successfully installs the complete
-    /// driver data, and the driver must use managed SR-IOV. The registration must be dropped before
-    /// anything its payload borrows and must not be forgotten. Probe must have exclusive access to
-    /// the PF registration slot. On a conventional PCI function, the registration remains
-    /// inactive.
-    pub unsafe fn new<'core, D>(
+    fn new_with_descriptor<'core, D, M>(
         pdev: &'a PciDevice<device::Core<'core>>,
         data: D,
-    ) -> impl PinInit<Self, Error> + use<'a, 'core, D, F>
+        make_descriptor: M,
+    ) -> impl PinInit<Self, Error> + use<'a, 'core, D, F, M>
     where
         D: PinInit<F::Of<'a>, Error> + 'a,
+        M: FnOnce(Pin<&F::Of<'a>>) -> Descriptor + 'a,
     {
         pin_init::pin_init_scope(move || {
             if pdev.is_virtfn() {
@@ -179,7 +188,7 @@ pub unsafe fn new<'core, D>(
 
             Ok(try_pin_init!(Self {
                 pdev,
-                inner <- VfRegistrationData::new(data),
+                inner <- VfRegistrationData::new(data, make_descriptor),
                 published,
                 _pin: PhantomPinned,
                 _: {
@@ -192,6 +201,53 @@ pub unsafe fn new<'core, D>(
             }))
         })
     }
+
+    /// Publishes typed PF data for bound VF drivers.
+    ///
+    /// This returns a pin-initializer so the registration and payload can be embedded directly in
+    /// the PF driver's pinned data.
+    ///
+    /// Initialization returns [`ENODEV`] for a VF and [`EBUSY`] if the PF has enabled VFs or
+    /// already has a registration.
+    ///
+    /// # Safety
+    ///
+    /// The caller must invoke this during the PCI driver's probe and embed the result in the driver
+    /// data. On an SR-IOV PF, no VF may be enabled before probe successfully installs the complete
+    /// driver data, and the driver must use managed SR-IOV. The registration must be dropped before
+    /// anything its payload borrows and must not be forgotten. Probe must have exclusive access to
+    /// the PF registration slot. On a conventional PCI function, the registration remains
+    /// inactive.
+    pub unsafe fn new<'core, D>(
+        pdev: &'a PciDevice<device::Core<'core>>,
+        data: D,
+    ) -> impl PinInit<Self, Error> + use<'a, 'core, D, F>
+    where
+        D: PinInit<F::Of<'a>, Error> + 'a,
+    {
+        Self::new_with_descriptor(pdev, data, |_| Descriptor::default())
+    }
+
+    /// Publishes typed PF data with an FFI operations table for C VF drivers.
+    ///
+    /// Rust VFs access the same payload through [`PciDevice::vf_registration_data()`] or
+    /// [`PciDevice::vf_registration_data_with()`].
+    ///
+    /// # Safety
+    ///
+    /// The caller must uphold the requirements of [`Self::new()`]. A C consumer must stop calling
+    /// and discard the borrow before returning from a failed VF probe, or before its VF remove
+    /// callback returns after a successful probe.
+    pub unsafe fn new_ffi<'core, A, D>(
+        pdev: &'a PciDevice<device::Core<'core>>,
+        data: D,
+    ) -> impl PinInit<Self, Error> + use<'a, 'core, A, D, F>
+    where
+        A: Abi<Context = F>,
+        D: PinInit<F::Of<'a>, Error> + 'a,
+    {
+        Self::new_with_descriptor(pdev, data, Descriptor::new::<A>)
+    }
 }
 
 #[pinned_drop]
@@ -251,9 +307,9 @@ unsafe fn vf_registration_data_pinned<F: ForLt + 'static>(&self) -> Result<Pin<&
             return Err(ENOENT);
         }
 
-        // SAFETY: The published pointer addresses a `VfRegistrationData`, whose first field is a
-        // `TypeId`.
-        let type_id = unsafe { ptr.cast::<TypeId>().read() };
+        // SAFETY: The published pointer addresses a `VfRegistrationData`, whose first field is
+        // its header.
+        let type_id = unsafe { (&raw const (*ptr.cast::<VfRegistrationHeader>()).type_id).read() };
         if type_id != TypeId::of::<F>() {
             return Err(EINVAL);
         }