mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Simon Song <xinmengs@nvidia.com>
To: <dakr@kernel.org>, <acourbot@nvidia.com>, <kvm@vger.kernel.org>,
	<rust-for-linux@vger.kernel.org>, <nova-gpu@lists.linux.dev>,
	<linux-pci@vger.kernel.org>
Cc: Zhi Wang <zhiw@nvidia.com>, <alex@shazbot.org>, <jgg@nvidia.com>,
	<yishaih@nvidia.com>, <skolothumtho@nvidia.com>,
	<kevin.tian@intel.com>, <airlied@gmail.com>, <simona@ffwll.ch>,
	<ojeda@kernel.org>, <alex.gaynor@gmail.com>,
	<boqun.feng@gmail.com>, <gary@garyguo.net>,
	<bjorn3_gh@protonmail.com>, <lossin@kernel.org>,
	<a.hindborg@kernel.org>, <aliceryhl@google.com>,
	<tmgross@umich.edu>, <jhubbard@nvidia.com>,
	<ecourtney@nvidia.com>, <cjia@nvidia.com>, <smitra@nvidia.com>,
	<kjaju@nvidia.com>, <alkumar@nvidia.com>, <ankita@nvidia.com>,
	<aniketa@nvidia.com>, <kwankhede@nvidia.com>,
	<targupta@nvidia.com>, <linux-kernel@vger.kernel.org>,
	<zhiwang@kernel.org>, <boqun@kernel.org>,
	<daniel.almeida@collabora.com>, <tamird@kernel.org>,
	<work@onurozkan.dev>, <jgg@ziepe.ca>, <bhelgaas@google.com>,
	<kwilczynski@kernel.org>, <dri-devel@lists.freedesktop.org>,
	<peter@colberg.org>, <xinmengs@nvidia.com>
Subject: [PATCH v3 8/9] gpu: nova-core: publish typed SR-IOV PF APIs
Date: Thu, 19 Nov 2026 13:50:43 -0800	[thread overview]
Message-ID: <20261119215046.2892839-9-xinmengs@nvidia.com> (raw)
In-Reply-To: <20261119215046.2892839-1-xinmengs@nvidia.com>

From: Zhi Wang <zhiw@nvidia.com>

Register NovaCoreVfApi as typed SR-IOV PF data once the GPU is initialized.

Enable and disable VFs through the sriov_enable() and sriov_disable()
callbacks. The VF registration removes VFs before dropping its PF API
data. Declare the registration before the GPU so its borrowed services
remain valid through VF teardown.

Expose a VF-side handle that borrows these services for each operation.
Return an owned instance on open and close it on drop while the VF
remains bound.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: Simon Song <xinmengs@nvidia.com>
---
 drivers/gpu/nova-core/driver.rs        |  34 ++++-
 drivers/gpu/nova-core/nova_core.rs     |   3 +-
 drivers/gpu/nova-core/vgpu.rs          |   1 -
 drivers/gpu/nova-core/vgpu/vgpu_api.rs | 178 +++++++++++++++++++------
 4 files changed, 171 insertions(+), 45 deletions(-)

diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 1785a5294823..a306e4233df6 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -29,7 +29,8 @@
         NovaCoreFwCtl,
         NovaCoreFwCtlData, //
     },
-    gpu::Gpu, //
+    gpu::Gpu,
+    vgpu::vgpu_api::NovaCoreVfApi, //
 };
 
 /// Counter for generating unique auxiliary device IDs.
@@ -41,6 +42,10 @@ pub(crate) struct NovaCore<'bound> {
     _reg: auxiliary::Registration<'bound, ForLt!(NovaCoreApi<'_>)>,
     /// Unregisters fwctl and drains its callbacks before GPU resources are dropped.
     _fwctl: fwctl::Registration<'bound, NovaCoreFwCtl>,
+    #[allow(clippy::type_complexity)]
+    #[pin]
+    #[uses('gpu: invariant)]
+    _vf_registration: pci::VfRegistration<'gpu, ForLt!(NovaCoreVfApi<'_>)>,
     #[pin]
     pub(crate) gpu: Gpu<'bound>,
     bar: pci::Bar<'bound, BAR0_SIZE>,
@@ -112,6 +117,10 @@ fn probe<'bound>(
 
         try_pin_init!(NovaCore {
             _: {
+                if pdev.is_virtfn() {
+                    return Err(ENODEV);
+                }
+
                 pdev.enable_device_mem()?;
                 pdev.set_master();
             },
@@ -150,6 +159,12 @@ fn probe<'bound>(
                 Ok(())
             }),
 
+            // SAFETY: The registration is stored in `NovaCore`, this driver's bus device
+            // private data.
+            _vf_registration <- unsafe {
+                pci::VfRegistration::new(pdev, NovaCoreVfApi::new(pdev, gpu))
+            },
+
             _reg: {
                 // TODO: Use `&gpu` self-referential pin-init syntax once available.
                 //
@@ -198,4 +213,21 @@ fn probe<'bound>(
             },
         })
     }
+
+    fn sriov_enable<'a>(
+        _dev: &'a pci::Device<Core<'_>>,
+        _this: Pin<&Self::Data<'a>>,
+        token: pci::SriovEnable<'a>,
+    ) -> Result<pci::SriovEnabled<'a>> {
+        token.enable_all()
+    }
+
+    fn sriov_disable<'a>(
+        _dev: &'a pci::Device<Core<'_>>,
+        _this: Pin<&Self::Data<'a>>,
+        token: pci::SriovDisable<'a>,
+    ) -> Result {
+        token.disable();
+        Ok(())
+    }
 }
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 5f09b69f3b54..a840b663cf73 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -31,9 +31,10 @@
 mod vbios;
 mod vgpu;
 
-#[cfg(CONFIG_PCI_IOV)]
 pub use vgpu::vgpu_api::{
     NovaCoreVfApi,
+    NovaCoreVfApiHandle,
+    VgpuInstance,
     VgpuTypeInfo, //
 };
 
diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs
index f198fa218297..38dcd68f52e7 100644
--- a/drivers/gpu/nova-core/vgpu.rs
+++ b/drivers/gpu/nova-core/vgpu.rs
@@ -42,7 +42,6 @@
 mod instance;
 mod log;
 mod scrubber;
-#[cfg_attr(not(CONFIG_PCI_IOV), expect(dead_code, unreachable_pub))]
 pub(crate) mod vgpu_api;
 mod vram;
 
diff --git a/drivers/gpu/nova-core/vgpu/vgpu_api.rs b/drivers/gpu/nova-core/vgpu/vgpu_api.rs
index f9f3e527bdc0..cfac0ca10c1e 100644
--- a/drivers/gpu/nova-core/vgpu/vgpu_api.rs
+++ b/drivers/gpu/nova-core/vgpu/vgpu_api.rs
@@ -8,7 +8,8 @@
 use kernel::{
     device,
     pci,
-    prelude::*, //
+    prelude::*,
+    types::ForLt, //
 };
 
 use crate::{
@@ -52,29 +53,41 @@ fn from_vgpu_type(vgpu_type: &VgpuType) -> Self {
 }
 
 /// PF-owned operations available while a VF driver is bound.
-pub struct NovaCoreVfApi<'gpu> {
+#[pin_data]
+pub struct NovaCoreVfApi<'gpu>
+where
+    // The GPU resources are invariant in their lifetime, which outlives this API borrow.
+    exists<'gpu_resources>: 'gpu,
+{
     pdev: &'gpu pci::Device<device::Bound>,
-    cmdq: &'gpu Cmdq<'gpu>,
-    bar: Bar0<'gpu>,
-    vgpu: Option<&'gpu VgpuManager<'gpu>>,
+    #[uses('gpu_resources: invariant)]
+    cmdq: &'gpu Cmdq<'gpu_resources>,
+    #[uses('gpu_resources)]
+    bar: Bar0<'gpu_resources>,
+    #[uses('gpu_resources: invariant)]
+    vgpu: Option<&'gpu VgpuManager<'gpu_resources>>,
     total_vfs: Option<NonZero<u16>>,
 }
 
 impl<'gpu> NovaCoreVfApi<'gpu> {
-    #[expect(dead_code)]
-    pub(crate) fn new(gpu: &'gpu Gpu<'gpu>, pdev: &'gpu pci::Device<device::Bound>) -> Self {
-        Self {
+    /// Creates the PF services that VF drivers access through the VF registration.
+    pub(crate) fn new<'res: 'gpu>(
+        pdev: &'gpu pci::Device<device::Bound>,
+        gpu: Pin<&'gpu Gpu<'res>>,
+    ) -> impl PinInit<Self, Error> + use<'gpu, 'res> {
+        let gpu = gpu.get_ref();
+        try_pin_init!(Self {
             pdev,
             cmdq: gpu.cmdq(),
             bar: gpu.bar0(),
             vgpu: gpu.vgpu_manager(),
             total_vfs: gpu.vgpu_total_vfs(),
-        }
+        })
     }
 
     /// Returns whether the PF booted with vGPU support enabled.
     pub fn is_available(&self) -> bool {
-        self.vgpu.is_some()
+        self.with_vgpu(|vgpu| vgpu.is_some())
     }
 
     fn gfid(&self, gfid: u32) -> Result<Gfid> {
@@ -84,11 +97,29 @@ fn gfid(&self, gfid: u32) -> Result<Gfid> {
 }
 
 impl NovaCoreVfApi<'_> {
+    /// Obtains the enabled PF services for a bound VF.
+    pub fn handle(vf: &pci::Device<device::Bound>) -> Result<NovaCoreVfApiHandle<'_>> {
+        let handle = NovaCoreVfApiHandle { vf };
+        handle.with(|api| {
+            if api.is_available() {
+                Ok(())
+            } else {
+                Err(ENODEV)
+            }
+        })?;
+        Ok(handle)
+    }
+
     /// Creates and boots an instance for a one-based VF ID.
     ///
     /// `sbdf` encodes the VF address as `(segment << 16) | (bus << 8) | devfn`.
     /// `vm_pid` identifies the VM process's thread group. This call may sleep.
-    pub fn open_instance(&self, gfid: u32, sbdf: u32, vm_pid: u32) -> Result<VgpuTypeInfo> {
+    pub fn open_instance(
+        self: Pin<&Self>,
+        gfid: u32,
+        sbdf: u32,
+        vm_pid: u32,
+    ) -> Result<VgpuTypeInfo> {
         let dev = self.pdev.as_ref();
         let gfid = self.gfid(gfid)?;
         let dbdf = Dbdf::from_raw(sbdf);
@@ -100,31 +131,33 @@ pub fn open_instance(&self, gfid: u32, sbdf: u32, vm_pid: u32) -> Result<VgpuTyp
             dbdf.into_raw()
         );
 
-        let bar = self.bar;
-        let cmdq = self.cmdq;
-        let vgpu = self.vgpu.ok_or(ENODEV)?;
-
-        let type_id = query_assigned_vf_type(dev, cmdq, dbdf)?;
-        dev_dbg!(
-            dev,
-            "vgpu_open: gfid={} assigned type_id={}\n",
-            gfid.get(),
-            type_id
-        );
-
-        let vgpu_type = query_vgpu_type(dev, cmdq, type_id)?;
-        dev_dbg!(
-            dev,
-            "vgpu_open: gfid={} vgpu_type={} fb_length={:#x}\n",
-            gfid.get(),
-            vgpu_type.vgpu_type_id,
-            vgpu_type.fb_length
-        );
-
-        let type_info = VgpuTypeInfo::from_vgpu_type(&vgpu_type);
-        vgpu.create_instance(bar, InstanceInfo::new(gfid, dbdf, vgpu_type, vm_pid))?;
-
-        Ok(type_info)
+        self.with_project_ref(|this| {
+            let bar = *this.bar;
+            let cmdq = *this.cmdq;
+            let vgpu = this.vgpu.ok_or(ENODEV)?;
+
+            let type_id = query_assigned_vf_type(dev, cmdq, dbdf)?;
+            dev_dbg!(
+                dev,
+                "vgpu_open: gfid={} assigned type_id={}\n",
+                gfid.get(),
+                type_id
+            );
+
+            let vgpu_type = query_vgpu_type(dev, cmdq, type_id)?;
+            dev_dbg!(
+                dev,
+                "vgpu_open: gfid={} vgpu_type={} fb_length={:#x}\n",
+                gfid.get(),
+                vgpu_type.vgpu_type_id,
+                vgpu_type.fb_length
+            );
+
+            let type_info = VgpuTypeInfo::from_vgpu_type(&vgpu_type);
+            vgpu.create_instance(bar, InstanceInfo::new(gfid, dbdf, vgpu_type, vm_pid))?;
+
+            Ok(type_info)
+        })
     }
 
     /// Tears down an instance, retaining resources if firmware cleanup fails.
@@ -136,11 +169,13 @@ pub fn close_instance(&self, gfid: u32) -> Result {
 
         dev_dbg!(dev, "vgpu_close: gfid={}\n", gfid.get());
 
-        let result = self.vgpu.ok_or(ENODEV)?.close_instance(gfid);
-        if let Err(error) = result {
-            dev_err!(dev, "vgpu_close: gfid={} failed: {:?}\n", gfid.get(), error);
-        }
-        result
+        self.with_vgpu(|vgpu| {
+            let result = vgpu.ok_or(ENODEV)?.close_instance(gfid);
+            if let Err(error) = result {
+                dev_err!(dev, "vgpu_close: gfid={} failed: {:?}\n", gfid.get(), error);
+            }
+            result
+        })
     }
 
     /// Resets an instance and scrubs its guest VRAM. This call may sleep.
@@ -150,9 +185,68 @@ pub fn reset_instance(&self, gfid: u32) -> Result {
 
         dev_dbg!(dev, "vgpu_reset: gfid={}\n", gfid.get());
 
-        self.vgpu.ok_or(ENODEV)?.reset_instance(gfid)?;
+        self.with_vgpu(|vgpu| vgpu.ok_or(ENODEV)?.reset_instance(gfid))?;
 
         dev_dbg!(dev, "vgpu_reset: gfid={} done\n", gfid.get());
         Ok(())
     }
 }
+
+/// Access to PF services for the lifetime of a VF driver binding.
+///
+/// The PF's registration keeps these services available until VF removal
+/// completes and is dropped before the GPU resources they borrow.
+pub struct NovaCoreVfApiHandle<'vf> {
+    vf: &'vf pci::Device<device::Bound>,
+}
+
+impl<'vf> NovaCoreVfApiHandle<'vf> {
+    /// Borrows the typed PF services for a single operation.
+    fn with<R>(
+        &self,
+        f: impl for<'borrow, 'data> FnOnce(Pin<&'borrow NovaCoreVfApi<'data>>) -> Result<R>,
+    ) -> Result<R> {
+        self.vf
+            .vf_registration_data_with::<ForLt!(NovaCoreVfApi<'_>), _>(f)?
+    }
+
+    /// Creates and boots an instance that closes when dropped.
+    ///
+    /// The arguments have the same meaning as in [`NovaCoreVfApi::open_instance`].
+    pub fn open(&self, gfid: u32, dbdf: u32, vm_pid: u32) -> Result<VgpuInstance<'vf>> {
+        let type_info = self.with(|api| api.open_instance(gfid, dbdf, vm_pid))?;
+        Ok(VgpuInstance {
+            api: Self { vf: self.vf },
+            gfid,
+            type_info,
+        })
+    }
+
+    /// Resets an active instance and scrubs its guest VRAM.
+    pub fn reset(&self, gfid: u32) -> Result {
+        self.with(|api| api.reset_instance(gfid))
+    }
+}
+
+/// An active instance whose teardown runs while its VF driver remains bound.
+///
+/// Dropping this guard may sleep while firmware teardown completes.
+pub struct VgpuInstance<'vf> {
+    api: NovaCoreVfApiHandle<'vf>,
+    gfid: u32,
+    type_info: VgpuTypeInfo,
+}
+
+impl VgpuInstance<'_> {
+    /// Returns the assigned PCI IDs and BAR1 aperture size.
+    #[inline]
+    pub fn type_info(&self) -> &VgpuTypeInfo {
+        &self.type_info
+    }
+}
+
+impl Drop for VgpuInstance<'_> {
+    fn drop(&mut self) {
+        let _ = self.api.with(|api| api.close_instance(self.gfid));
+    }
+}
-- 
2.43.7


  parent reply	other threads:[~2026-11-19 21:52 UTC|newest]

Thread overview: 10+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-19 21:50 [PATCH v3 0/9] Rust VFIO PCI support and NVIDIA vGPU driver Simon Song
2026-11-19 21:50 ` [PATCH v3 1/9] rust: pci: add VFIO override device IDs Simon Song
2026-11-19 21:50 ` [PATCH v3 2/9] rust: pci: expose the VF index within its PF Simon Song
2026-11-19 21:50 ` [PATCH v3 3/9] rust: pci: allow drivers to manage DMA ownership Simon Song
2026-11-19 21:50 ` [PATCH v3 4/9] rust: pci: add reset callbacks Simon Song
2026-11-19 21:50 ` [PATCH v3 5/9] rust: pci: add an error_detected callback Simon Song
2026-11-19 21:50 ` [PATCH v3 6/9] rust: vfio: add common VFIO device handling Simon Song
2026-11-19 21:50 ` [PATCH v3 7/9] rust: vfio: add PCI variant driver abstractions Simon Song
2026-11-19 21:50 ` Simon Song [this message]
2026-11-19 21:50 ` [PATCH v3 9/9] vfio/nova-vgpu: add the Rust VFIO variant driver Simon Song

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261119215046.2892839-9-xinmengs@nvidia.com \
    --to=xinmengs@nvidia.com \
    --cc=a.hindborg@kernel.org \
    --cc=acourbot@nvidia.com \
    --cc=airlied@gmail.com \
    --cc=alex.gaynor@gmail.com \
    --cc=alex@shazbot.org \
    --cc=aliceryhl@google.com \
    --cc=alkumar@nvidia.com \
    --cc=aniketa@nvidia.com \
    --cc=ankita@nvidia.com \
    --cc=bhelgaas@google.com \
    --cc=bjorn3_gh@protonmail.com \
    --cc=boqun.feng@gmail.com \
    --cc=boqun@kernel.org \
    --cc=cjia@nvidia.com \
    --cc=dakr@kernel.org \
    --cc=daniel.almeida@collabora.com \
    --cc=dri-devel@lists.freedesktop.org \
    --cc=ecourtney@nvidia.com \
    --cc=gary@garyguo.net \
    --cc=jgg@nvidia.com \
    --cc=jgg@ziepe.ca \
    --cc=jhubbard@nvidia.com \
    --cc=kevin.tian@intel.com \
    --cc=kjaju@nvidia.com \
    --cc=kvm@vger.kernel.org \
    --cc=kwankhede@nvidia.com \
    --cc=kwilczynski@kernel.org \
    --cc=linux-kernel@vger.kernel.org \
    --cc=linux-pci@vger.kernel.org \
    --cc=lossin@kernel.org \
    --cc=nova-gpu@lists.linux.dev \
    --cc=ojeda@kernel.org \
    --cc=peter@colberg.org \
    --cc=rust-for-linux@vger.kernel.org \
    --cc=simona@ffwll.ch \
    --cc=skolothumtho@nvidia.com \
    --cc=smitra@nvidia.com \
    --cc=tamird@kernel.org \
    --cc=targupta@nvidia.com \
    --cc=tmgross@umich.edu \
    --cc=work@onurozkan.dev \
    --cc=yishaih@nvidia.com \
    --cc=zhiw@nvidia.com \
    --cc=zhiwang@kernel.org \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®