From: Simon Song <xinmengs@nvidia.com>
To: <dakr@kernel.org>, <acourbot@nvidia.com>, <kvm@vger.kernel.org>,
<rust-for-linux@vger.kernel.org>, <nova-gpu@lists.linux.dev>,
<linux-pci@vger.kernel.org>
Cc: Zhi Wang <zhiw@nvidia.com>, <alex@shazbot.org>, <jgg@nvidia.com>,
<yishaih@nvidia.com>, <skolothumtho@nvidia.com>,
<kevin.tian@intel.com>, <airlied@gmail.com>, <simona@ffwll.ch>,
<ojeda@kernel.org>, <alex.gaynor@gmail.com>,
<boqun.feng@gmail.com>, <gary@garyguo.net>,
<bjorn3_gh@protonmail.com>, <lossin@kernel.org>,
<a.hindborg@kernel.org>, <aliceryhl@google.com>,
<tmgross@umich.edu>, <jhubbard@nvidia.com>,
<ecourtney@nvidia.com>, <cjia@nvidia.com>, <smitra@nvidia.com>,
<kjaju@nvidia.com>, <alkumar@nvidia.com>, <ankita@nvidia.com>,
<aniketa@nvidia.com>, <kwankhede@nvidia.com>,
<targupta@nvidia.com>, <linux-kernel@vger.kernel.org>,
<zhiwang@kernel.org>, <boqun@kernel.org>,
<daniel.almeida@collabora.com>, <tamird@kernel.org>,
<work@onurozkan.dev>, <jgg@ziepe.ca>, <bhelgaas@google.com>,
<kwilczynski@kernel.org>, <dri-devel@lists.freedesktop.org>,
<peter@colberg.org>, <xinmengs@nvidia.com>
Subject: [PATCH v3 8/9] gpu: nova-core: publish typed SR-IOV PF APIs
Date: Thu, 19 Nov 2026 13:50:43 -0800 [thread overview]
Message-ID: <20261119215046.2892839-9-xinmengs@nvidia.com> (raw)
In-Reply-To: <20261119215046.2892839-1-xinmengs@nvidia.com>
From: Zhi Wang <zhiw@nvidia.com>
Register NovaCoreVfApi as typed SR-IOV PF data once the GPU is initialized.
Enable and disable VFs through the sriov_enable() and sriov_disable()
callbacks. The VF registration removes VFs before dropping its PF API
data. Declare the registration before the GPU so its borrowed services
remain valid through VF teardown.
Expose a VF-side handle that borrows these services for each operation.
Return an owned instance on open and close it on drop while the VF
remains bound.
Signed-off-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: Simon Song <xinmengs@nvidia.com>
---
drivers/gpu/nova-core/driver.rs | 34 ++++-
drivers/gpu/nova-core/nova_core.rs | 3 +-
drivers/gpu/nova-core/vgpu.rs | 1 -
drivers/gpu/nova-core/vgpu/vgpu_api.rs | 178 +++++++++++++++++++------
4 files changed, 171 insertions(+), 45 deletions(-)
diff --git a/drivers/gpu/nova-core/driver.rs b/drivers/gpu/nova-core/driver.rs
index 1785a5294823..a306e4233df6 100644
--- a/drivers/gpu/nova-core/driver.rs
+++ b/drivers/gpu/nova-core/driver.rs
@@ -29,7 +29,8 @@
NovaCoreFwCtl,
NovaCoreFwCtlData, //
},
- gpu::Gpu, //
+ gpu::Gpu,
+ vgpu::vgpu_api::NovaCoreVfApi, //
};
/// Counter for generating unique auxiliary device IDs.
@@ -41,6 +42,10 @@ pub(crate) struct NovaCore<'bound> {
_reg: auxiliary::Registration<'bound, ForLt!(NovaCoreApi<'_>)>,
/// Unregisters fwctl and drains its callbacks before GPU resources are dropped.
_fwctl: fwctl::Registration<'bound, NovaCoreFwCtl>,
+ #[allow(clippy::type_complexity)]
+ #[pin]
+ #[uses('gpu: invariant)]
+ _vf_registration: pci::VfRegistration<'gpu, ForLt!(NovaCoreVfApi<'_>)>,
#[pin]
pub(crate) gpu: Gpu<'bound>,
bar: pci::Bar<'bound, BAR0_SIZE>,
@@ -112,6 +117,10 @@ fn probe<'bound>(
try_pin_init!(NovaCore {
_: {
+ if pdev.is_virtfn() {
+ return Err(ENODEV);
+ }
+
pdev.enable_device_mem()?;
pdev.set_master();
},
@@ -150,6 +159,12 @@ fn probe<'bound>(
Ok(())
}),
+ // SAFETY: The registration is stored in `NovaCore`, this driver's bus device
+ // private data.
+ _vf_registration <- unsafe {
+ pci::VfRegistration::new(pdev, NovaCoreVfApi::new(pdev, gpu))
+ },
+
_reg: {
// TODO: Use `&gpu` self-referential pin-init syntax once available.
//
@@ -198,4 +213,21 @@ fn probe<'bound>(
},
})
}
+
+ fn sriov_enable<'a>(
+ _dev: &'a pci::Device<Core<'_>>,
+ _this: Pin<&Self::Data<'a>>,
+ token: pci::SriovEnable<'a>,
+ ) -> Result<pci::SriovEnabled<'a>> {
+ token.enable_all()
+ }
+
+ fn sriov_disable<'a>(
+ _dev: &'a pci::Device<Core<'_>>,
+ _this: Pin<&Self::Data<'a>>,
+ token: pci::SriovDisable<'a>,
+ ) -> Result {
+ token.disable();
+ Ok(())
+ }
}
diff --git a/drivers/gpu/nova-core/nova_core.rs b/drivers/gpu/nova-core/nova_core.rs
index 5f09b69f3b54..a840b663cf73 100644
--- a/drivers/gpu/nova-core/nova_core.rs
+++ b/drivers/gpu/nova-core/nova_core.rs
@@ -31,9 +31,10 @@
mod vbios;
mod vgpu;
-#[cfg(CONFIG_PCI_IOV)]
pub use vgpu::vgpu_api::{
NovaCoreVfApi,
+ NovaCoreVfApiHandle,
+ VgpuInstance,
VgpuTypeInfo, //
};
diff --git a/drivers/gpu/nova-core/vgpu.rs b/drivers/gpu/nova-core/vgpu.rs
index f198fa218297..38dcd68f52e7 100644
--- a/drivers/gpu/nova-core/vgpu.rs
+++ b/drivers/gpu/nova-core/vgpu.rs
@@ -42,7 +42,6 @@
mod instance;
mod log;
mod scrubber;
-#[cfg_attr(not(CONFIG_PCI_IOV), expect(dead_code, unreachable_pub))]
pub(crate) mod vgpu_api;
mod vram;
diff --git a/drivers/gpu/nova-core/vgpu/vgpu_api.rs b/drivers/gpu/nova-core/vgpu/vgpu_api.rs
index f9f3e527bdc0..cfac0ca10c1e 100644
--- a/drivers/gpu/nova-core/vgpu/vgpu_api.rs
+++ b/drivers/gpu/nova-core/vgpu/vgpu_api.rs
@@ -8,7 +8,8 @@
use kernel::{
device,
pci,
- prelude::*, //
+ prelude::*,
+ types::ForLt, //
};
use crate::{
@@ -52,29 +53,41 @@ fn from_vgpu_type(vgpu_type: &VgpuType) -> Self {
}
/// PF-owned operations available while a VF driver is bound.
-pub struct NovaCoreVfApi<'gpu> {
+#[pin_data]
+pub struct NovaCoreVfApi<'gpu>
+where
+ // The GPU resources are invariant in their lifetime, which outlives this API borrow.
+ exists<'gpu_resources>: 'gpu,
+{
pdev: &'gpu pci::Device<device::Bound>,
- cmdq: &'gpu Cmdq<'gpu>,
- bar: Bar0<'gpu>,
- vgpu: Option<&'gpu VgpuManager<'gpu>>,
+ #[uses('gpu_resources: invariant)]
+ cmdq: &'gpu Cmdq<'gpu_resources>,
+ #[uses('gpu_resources)]
+ bar: Bar0<'gpu_resources>,
+ #[uses('gpu_resources: invariant)]
+ vgpu: Option<&'gpu VgpuManager<'gpu_resources>>,
total_vfs: Option<NonZero<u16>>,
}
impl<'gpu> NovaCoreVfApi<'gpu> {
- #[expect(dead_code)]
- pub(crate) fn new(gpu: &'gpu Gpu<'gpu>, pdev: &'gpu pci::Device<device::Bound>) -> Self {
- Self {
+ /// Creates the PF services that VF drivers access through the VF registration.
+ pub(crate) fn new<'res: 'gpu>(
+ pdev: &'gpu pci::Device<device::Bound>,
+ gpu: Pin<&'gpu Gpu<'res>>,
+ ) -> impl PinInit<Self, Error> + use<'gpu, 'res> {
+ let gpu = gpu.get_ref();
+ try_pin_init!(Self {
pdev,
cmdq: gpu.cmdq(),
bar: gpu.bar0(),
vgpu: gpu.vgpu_manager(),
total_vfs: gpu.vgpu_total_vfs(),
- }
+ })
}
/// Returns whether the PF booted with vGPU support enabled.
pub fn is_available(&self) -> bool {
- self.vgpu.is_some()
+ self.with_vgpu(|vgpu| vgpu.is_some())
}
fn gfid(&self, gfid: u32) -> Result<Gfid> {
@@ -84,11 +97,29 @@ fn gfid(&self, gfid: u32) -> Result<Gfid> {
}
impl NovaCoreVfApi<'_> {
+ /// Obtains the enabled PF services for a bound VF.
+ pub fn handle(vf: &pci::Device<device::Bound>) -> Result<NovaCoreVfApiHandle<'_>> {
+ let handle = NovaCoreVfApiHandle { vf };
+ handle.with(|api| {
+ if api.is_available() {
+ Ok(())
+ } else {
+ Err(ENODEV)
+ }
+ })?;
+ Ok(handle)
+ }
+
/// Creates and boots an instance for a one-based VF ID.
///
/// `sbdf` encodes the VF address as `(segment << 16) | (bus << 8) | devfn`.
/// `vm_pid` identifies the VM process's thread group. This call may sleep.
- pub fn open_instance(&self, gfid: u32, sbdf: u32, vm_pid: u32) -> Result<VgpuTypeInfo> {
+ pub fn open_instance(
+ self: Pin<&Self>,
+ gfid: u32,
+ sbdf: u32,
+ vm_pid: u32,
+ ) -> Result<VgpuTypeInfo> {
let dev = self.pdev.as_ref();
let gfid = self.gfid(gfid)?;
let dbdf = Dbdf::from_raw(sbdf);
@@ -100,31 +131,33 @@ pub fn open_instance(&self, gfid: u32, sbdf: u32, vm_pid: u32) -> Result<VgpuTyp
dbdf.into_raw()
);
- let bar = self.bar;
- let cmdq = self.cmdq;
- let vgpu = self.vgpu.ok_or(ENODEV)?;
-
- let type_id = query_assigned_vf_type(dev, cmdq, dbdf)?;
- dev_dbg!(
- dev,
- "vgpu_open: gfid={} assigned type_id={}\n",
- gfid.get(),
- type_id
- );
-
- let vgpu_type = query_vgpu_type(dev, cmdq, type_id)?;
- dev_dbg!(
- dev,
- "vgpu_open: gfid={} vgpu_type={} fb_length={:#x}\n",
- gfid.get(),
- vgpu_type.vgpu_type_id,
- vgpu_type.fb_length
- );
-
- let type_info = VgpuTypeInfo::from_vgpu_type(&vgpu_type);
- vgpu.create_instance(bar, InstanceInfo::new(gfid, dbdf, vgpu_type, vm_pid))?;
-
- Ok(type_info)
+ self.with_project_ref(|this| {
+ let bar = *this.bar;
+ let cmdq = *this.cmdq;
+ let vgpu = this.vgpu.ok_or(ENODEV)?;
+
+ let type_id = query_assigned_vf_type(dev, cmdq, dbdf)?;
+ dev_dbg!(
+ dev,
+ "vgpu_open: gfid={} assigned type_id={}\n",
+ gfid.get(),
+ type_id
+ );
+
+ let vgpu_type = query_vgpu_type(dev, cmdq, type_id)?;
+ dev_dbg!(
+ dev,
+ "vgpu_open: gfid={} vgpu_type={} fb_length={:#x}\n",
+ gfid.get(),
+ vgpu_type.vgpu_type_id,
+ vgpu_type.fb_length
+ );
+
+ let type_info = VgpuTypeInfo::from_vgpu_type(&vgpu_type);
+ vgpu.create_instance(bar, InstanceInfo::new(gfid, dbdf, vgpu_type, vm_pid))?;
+
+ Ok(type_info)
+ })
}
/// Tears down an instance, retaining resources if firmware cleanup fails.
@@ -136,11 +169,13 @@ pub fn close_instance(&self, gfid: u32) -> Result {
dev_dbg!(dev, "vgpu_close: gfid={}\n", gfid.get());
- let result = self.vgpu.ok_or(ENODEV)?.close_instance(gfid);
- if let Err(error) = result {
- dev_err!(dev, "vgpu_close: gfid={} failed: {:?}\n", gfid.get(), error);
- }
- result
+ self.with_vgpu(|vgpu| {
+ let result = vgpu.ok_or(ENODEV)?.close_instance(gfid);
+ if let Err(error) = result {
+ dev_err!(dev, "vgpu_close: gfid={} failed: {:?}\n", gfid.get(), error);
+ }
+ result
+ })
}
/// Resets an instance and scrubs its guest VRAM. This call may sleep.
@@ -150,9 +185,68 @@ pub fn reset_instance(&self, gfid: u32) -> Result {
dev_dbg!(dev, "vgpu_reset: gfid={}\n", gfid.get());
- self.vgpu.ok_or(ENODEV)?.reset_instance(gfid)?;
+ self.with_vgpu(|vgpu| vgpu.ok_or(ENODEV)?.reset_instance(gfid))?;
dev_dbg!(dev, "vgpu_reset: gfid={} done\n", gfid.get());
Ok(())
}
}
+
+/// Access to PF services for the lifetime of a VF driver binding.
+///
+/// The PF's registration keeps these services available until VF removal
+/// completes and is dropped before the GPU resources they borrow.
+pub struct NovaCoreVfApiHandle<'vf> {
+ vf: &'vf pci::Device<device::Bound>,
+}
+
+impl<'vf> NovaCoreVfApiHandle<'vf> {
+ /// Borrows the typed PF services for a single operation.
+ fn with<R>(
+ &self,
+ f: impl for<'borrow, 'data> FnOnce(Pin<&'borrow NovaCoreVfApi<'data>>) -> Result<R>,
+ ) -> Result<R> {
+ self.vf
+ .vf_registration_data_with::<ForLt!(NovaCoreVfApi<'_>), _>(f)?
+ }
+
+ /// Creates and boots an instance that closes when dropped.
+ ///
+ /// The arguments have the same meaning as in [`NovaCoreVfApi::open_instance`].
+ pub fn open(&self, gfid: u32, dbdf: u32, vm_pid: u32) -> Result<VgpuInstance<'vf>> {
+ let type_info = self.with(|api| api.open_instance(gfid, dbdf, vm_pid))?;
+ Ok(VgpuInstance {
+ api: Self { vf: self.vf },
+ gfid,
+ type_info,
+ })
+ }
+
+ /// Resets an active instance and scrubs its guest VRAM.
+ pub fn reset(&self, gfid: u32) -> Result {
+ self.with(|api| api.reset_instance(gfid))
+ }
+}
+
+/// An active instance whose teardown runs while its VF driver remains bound.
+///
+/// Dropping this guard may sleep while firmware teardown completes.
+pub struct VgpuInstance<'vf> {
+ api: NovaCoreVfApiHandle<'vf>,
+ gfid: u32,
+ type_info: VgpuTypeInfo,
+}
+
+impl VgpuInstance<'_> {
+ /// Returns the assigned PCI IDs and BAR1 aperture size.
+ #[inline]
+ pub fn type_info(&self) -> &VgpuTypeInfo {
+ &self.type_info
+ }
+}
+
+impl Drop for VgpuInstance<'_> {
+ fn drop(&mut self) {
+ let _ = self.api.with(|api| api.close_instance(self.gfid));
+ }
+}
--
2.43.7
next prev parent reply other threads:[~2026-11-19 21:52 UTC|newest]
Thread overview: 10+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-11-19 21:50 [PATCH v3 0/9] Rust VFIO PCI support and NVIDIA vGPU driver Simon Song
2026-11-19 21:50 ` [PATCH v3 1/9] rust: pci: add VFIO override device IDs Simon Song
2026-11-19 21:50 ` [PATCH v3 2/9] rust: pci: expose the VF index within its PF Simon Song
2026-11-19 21:50 ` [PATCH v3 3/9] rust: pci: allow drivers to manage DMA ownership Simon Song
2026-11-19 21:50 ` [PATCH v3 4/9] rust: pci: add reset callbacks Simon Song
2026-11-19 21:50 ` [PATCH v3 5/9] rust: pci: add an error_detected callback Simon Song
2026-11-19 21:50 ` [PATCH v3 6/9] rust: vfio: add common VFIO device handling Simon Song
2026-11-19 21:50 ` [PATCH v3 7/9] rust: vfio: add PCI variant driver abstractions Simon Song
2026-11-19 21:50 ` Simon Song [this message]
2026-11-19 21:50 ` [PATCH v3 9/9] vfio/nova-vgpu: add the Rust VFIO variant driver Simon Song
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20261119215046.2892839-9-xinmengs@nvidia.com \
--to=xinmengs@nvidia.com \
--cc=a.hindborg@kernel.org \
--cc=acourbot@nvidia.com \
--cc=airlied@gmail.com \
--cc=alex.gaynor@gmail.com \
--cc=alex@shazbot.org \
--cc=aliceryhl@google.com \
--cc=alkumar@nvidia.com \
--cc=aniketa@nvidia.com \
--cc=ankita@nvidia.com \
--cc=bhelgaas@google.com \
--cc=bjorn3_gh@protonmail.com \
--cc=boqun.feng@gmail.com \
--cc=boqun@kernel.org \
--cc=cjia@nvidia.com \
--cc=dakr@kernel.org \
--cc=daniel.almeida@collabora.com \
--cc=dri-devel@lists.freedesktop.org \
--cc=ecourtney@nvidia.com \
--cc=gary@garyguo.net \
--cc=jgg@nvidia.com \
--cc=jgg@ziepe.ca \
--cc=jhubbard@nvidia.com \
--cc=kevin.tian@intel.com \
--cc=kjaju@nvidia.com \
--cc=kvm@vger.kernel.org \
--cc=kwankhede@nvidia.com \
--cc=kwilczynski@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-pci@vger.kernel.org \
--cc=lossin@kernel.org \
--cc=nova-gpu@lists.linux.dev \
--cc=ojeda@kernel.org \
--cc=peter@colberg.org \
--cc=rust-for-linux@vger.kernel.org \
--cc=simona@ffwll.ch \
--cc=skolothumtho@nvidia.com \
--cc=smitra@nvidia.com \
--cc=tamird@kernel.org \
--cc=targupta@nvidia.com \
--cc=tmgross@umich.edu \
--cc=work@onurozkan.dev \
--cc=yishaih@nvidia.com \
--cc=zhiw@nvidia.com \
--cc=zhiwang@kernel.org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox
all inboxes | Powered by JetHome®