[PATCH 5/7] rust: vfio: separate common device handling from PCI

[Zhi Wang <zhiw@nvidia.com>]

A VFIO PCI device contains a VFIO device. Expose that relationship in
Rust so common device handling is independent of PCI resources.

Add vfio::Device and have vfio::pci::Device borrow its embedded device
through AsRef, preserving the callback context. Share reference counting
and callback-data lifetime handling in vfio. Keep driver operations,
allocation, registration and the PCI enable/close sequence in vfio::pci.

No functional change is intended.

Signed-off-by: Zhi Wang <zhiw@nvidia.com>
---
 rust/kernel/vfio.rs     | 200 +++++++++++++++++++++++++++++++++++++-
 rust/kernel/vfio/pci.rs | 207 +++++++++++++---------------------------
 2 files changed, 263 insertions(+), 144 deletions(-)

diff --git a/rust/kernel/vfio.rs b/rust/kernel/vfio.rs
index ed93066e2da5..a0bfbc7c59a0 100644
--- a/rust/kernel/vfio.rs
+++ b/rust/kernel/vfio.rs
@@ -1,14 +1,29 @@
 // SPDX-License-Identifier: GPL-2.0
 
-//! Virtual Function I/O (VFIO) abstractions.
+//! Virtual Function I/O (VFIO) devices and callback data.
+//!
+//! Bus-specific device types borrow their embedded VFIO device through [`AsRef`].
+//! C header: [`include/linux/vfio.h`](srctree/include/linux/vfio.h)
 
 use crate::{
     bindings,
     prelude::*,
+    sync::aref::AlwaysRefCounted,
+    types::{
+        CovariantForLt,
+        ForLt,
+        NotThreadSafe,
+        Opaque, //
+    },
     uaccess::{
         UserPtr,
         UserSlice, //
-    },
+    }, //
+};
+use core::{
+    cell::UnsafeCell,
+    marker::PhantomData,
+    ptr::NonNull, //
 };
 
 #[cfg(CONFIG_VFIO_PCI_CORE)]
@@ -17,12 +32,187 @@
 /// Reset the VFIO device.
 pub const DEVICE_RESET: u32 = bindings::VFIO_DEVICE_RESET;
 
-/// Reset the PCI slot or bus containing a VFIO device.
-pub const DEVICE_PCI_HOT_RESET: u32 = bindings::VFIO_DEVICE_PCI_HOT_RESET;
+/// The context in which a VFIO device reference is valid.
+///
+/// Callback views can only be borrowed from the corresponding VFIO trampoline.
+/// They cannot be refcounted or shared across threads.
+pub trait DeviceContext: private::Sealed {}
+
+/// An ordinary device reference, without callback-specific operations.
+pub struct Normal;
+
+/// The device is undergoing its exclusive first-open callback.
+pub struct Open;
+
+/// The device is borrowed for a VFIO ioctl callback on the current thread.
+pub struct Ioctl;
+
+/// The device is borrowed for a VFIO read callback on the current thread.
+pub struct Read;
+
+/// The device is borrowed for a VFIO write callback on the current thread.
+pub struct Write;
+
+/// The device is borrowed for a VFIO mmap callback on the current thread.
+pub struct Mmap;
+
+/// The device is borrowed for a VFIO region-info callback on the current thread.
+pub struct GetRegionInfo;
+
+mod private {
+    pub trait Sealed {}
+
+    impl Sealed for super::Normal {}
+    impl Sealed for super::Open {}
+    impl Sealed for super::Write {}
+    impl Sealed for super::Mmap {}
+    impl Sealed for super::Ioctl {}
+    impl Sealed for super::Read {}
+    impl Sealed for super::GetRegionInfo {}
+}
+
+impl DeviceContext for Normal {}
+impl DeviceContext for Open {}
+impl DeviceContext for Write {}
+impl DeviceContext for Mmap {}
+impl DeviceContext for Ioctl {}
+impl DeviceContext for Read {}
+impl DeviceContext for GetRegionInfo {}
+
+/// A VFIO device, independent of its bus-specific wrapper.
+///
+/// # Invariants
+///
+/// The underlying `vfio_device` is initialized and remains alive while borrowed.
+/// Callback contexts are borrowed only for the corresponding callback and thread;
+/// only [`Normal`] references can be reference-counted.
+#[repr(transparent)]
+pub struct Device<Ctx: DeviceContext = Normal> {
+    raw: Opaque<bindings::vfio_device>,
+    _context: PhantomData<(Ctx, NotThreadSafe)>,
+}
+
+impl<Ctx: DeviceContext> Device<Ctx> {
+    /// # Safety
+    ///
+    /// `raw` must point to an initialized `vfio_device` that remains alive for
+    /// `'a`. The caller must provide the guarantees of `Ctx` throughout the borrow.
+    #[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
+    unsafe fn from_raw<'a>(raw: *mut bindings::vfio_device) -> &'a Self {
+        // SAFETY: Self is a transparent wrapper; the caller guarantees validity and context.
+        unsafe { &*raw.cast() }
+    }
+
+    fn as_raw(&self) -> *mut bindings::vfio_device {
+        self.raw.get()
+    }
+}
+
+// SAFETY: VFIO uses the embedded class device's refcount to own the full allocation.
+unsafe impl AlwaysRefCounted for Device {
+    fn inc_ref(&self) {
+        // SAFETY: A shared reference keeps the initialized VFIO device alive.
+        unsafe { bindings::get_device(&raw mut (*self.as_raw()).device) };
+    }
+
+    unsafe fn dec_ref(obj: NonNull<Self>) {
+        // SAFETY: The caller owns the VFIO reference being released.
+        unsafe { bindings::put_device(&raw mut (*obj.as_ref().as_raw()).device) };
+    }
+}
+
+// SAFETY: Ordinary VFIO references expose no unsynchronized callback operations.
+unsafe impl Send for Device {}
+// SAFETY: Shared access only changes the embedded class device's reference count.
+unsafe impl Sync for Device {}
+
+/// Callback data stored after the bus-specific C device structure.
+///
+/// # Invariants
+///
+/// `reg_data` is dangling when unpublished; otherwise it borrows pinned data kept
+/// alive until callback access has ended. `open_data` is null outside a successful
+/// first open, otherwise owns a pinned allocation until the last close.
+/// The bus-specific layer drains callbacks before releasing either allocation.
+struct CallbackData<R: CovariantForLt, O: ForLt> {
+    reg_data: UnsafeCell<NonNull<R::Of<'static>>>,
+    open_data: UnsafeCell<*mut O::Of<'static>>,
+}
+
+#[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
+impl<R: CovariantForLt, O: ForLt> CallbackData<R, O> {
+    fn new() -> Self {
+        Self {
+            reg_data: UnsafeCell::new(NonNull::dangling()),
+            open_data: UnsafeCell::new(core::ptr::null_mut()),
+        }
+    }
+
+    /// Access registration data without allowing its erased lifetime to escape.
+    ///
+    /// # Safety
+    ///
+    /// Registration data must be published and remain valid through this call.
+    /// The caller must exclude concurrent changes to `reg_data`.
+    unsafe fn registration_data_with<V>(&self, f: impl for<'a> FnOnce(&'a R::Of<'a>) -> V) -> V {
+        // SAFETY: The caller keeps registration data alive and excludes mutation.
+        // Covariance permits shortening its erased lifetime to this borrow.
+        let reg_data = unsafe { (*self.reg_data.get()).cast::<R::Of<'_>>().as_ref() };
+        f(reg_data)
+    }
+
+    /// # Safety
+    ///
+    /// The caller must be an I/O callback while VFIO keeps the device open.
+    /// Registration and open data must remain valid, with their pointer slots
+    /// unchanged throughout the call.
+    unsafe fn callback_data_with<V>(
+        &self,
+        f: impl for<'borrow, 'data> FnOnce(&'borrow R::Of<'data>, Pin<&'borrow O::Of<'data>>) -> V,
+    ) -> V {
+        // SAFETY: The caller excludes open/close and unregister while borrowing the data.
+        unsafe {
+            self.registration_data_with(|reg_data| {
+                let ptr = (*self.open_data.get()).cast::<O::Of<'_>>();
+                // The data lifetime stays independent of the callback borrow,
+                // so borrowed OpenData fields cannot be stored back into OpenData.
+                f(reg_data, Pin::new_unchecked(&*ptr))
+            })
+        }
+    }
+
+    /// # Safety
+    ///
+    /// Call only during exclusive first open, with no existing open data.
+    /// All resources borrowed for `'a` must remain valid until `close()` returns.
+    unsafe fn open<'a, I: PinInit<O::Of<'a>, Error>>(&self, init: impl FnOnce() -> I) -> Result {
+        // Allocate before calling the driver so allocation failure cannot follow open.
+        let data = KBox::<O::Of<'a>>::new_uninit(GFP_KERNEL)?;
+        let data = data.write_pin_init(init())?;
+
+        // SAFETY: Only the owning pointer moves; close drops the allocation in place.
+        let raw = KBox::into_raw(unsafe { Pin::into_inner_unchecked(data) });
+        // SAFETY: First open is exclusive; the caller keeps borrowed resources alive
+        // through close. Lifetimes do not affect the allocation's layout.
+        unsafe { *self.open_data.get() = raw.cast::<O::Of<'static>>() };
+        Ok(())
+    }
+
+    /// # Safety
+    ///
+    /// Call once after a successful open, with all callbacks drained and the next
+    /// open excluded. Registration data and borrowed resources must remain alive.
+    unsafe fn close(&self) {
+        // SAFETY: Last close exclusively takes the allocation published by open.
+        let raw = unsafe { core::mem::replace(&mut *self.open_data.get(), core::ptr::null_mut()) };
+        // SAFETY: The allocation came from KBox::into_raw and is destroyed without moving it.
+        unsafe { drop(KBox::from_raw(raw)) };
+    }
+}
 
 /// Capability buffer supplied by VFIO for a region-info callback.
-#[cfg(CONFIG_VFIO_PCI_CORE)]
 pub struct InfoCap<'a> {
+    #[cfg_attr(not(CONFIG_VFIO_PCI_CORE), expect(dead_code))]
     raw: &'a mut bindings::vfio_info_cap,
 }
 
diff --git a/rust/kernel/vfio/pci.rs b/rust/kernel/vfio/pci.rs
index 0fca288fe6ae..20a89a113751 100644
--- a/rust/kernel/vfio/pci.rs
+++ b/rust/kernel/vfio/pci.rs
@@ -10,9 +10,20 @@
 //! C header: [`include/linux/vfio_pci_core.h`](srctree/include/linux/vfio_pci_core.h)
 
 use super::{
+    CallbackData,
     InfoCap,
     UserBuf, //
 };
+pub use super::{
+    DeviceContext,
+    GetRegionInfo,
+    Ioctl,
+    Mmap,
+    Normal,
+    Open,
+    Read,
+    Write, //
+};
 
 use crate::{
     alloc::allocator::Kmalloc,
@@ -43,61 +54,16 @@
 };
 use core::{
     alloc::Layout,
-    cell::UnsafeCell,
     marker::PhantomData,
     ptr::NonNull, //
 };
 
+/// Reset the PCI slot or bus containing a VFIO device.
+pub const DEVICE_PCI_HOT_RESET: u32 = bindings::VFIO_DEVICE_PCI_HOT_RESET;
+
 /// Index of the PCI configuration-space region.
 pub const CONFIG_REGION_INDEX: u32 = bindings::VFIO_PCI_CONFIG_REGION_INDEX;
 
-/// The context in which a VFIO PCI device reference is valid.
-///
-/// Callback views can only be borrowed from the corresponding VFIO trampoline.
-/// They cannot be refcounted or shared across threads.
-pub trait DeviceContext: private::Sealed {}
-
-/// An ordinary device reference, without callback-specific operations.
-pub struct Normal;
-
-/// The device is enabled but has not yet completed its first-open callback.
-pub struct Open;
-
-/// The device is borrowed for a VFIO ioctl callback on the current thread.
-pub struct Ioctl;
-
-/// The device is borrowed for a VFIO read callback on the current thread.
-pub struct Read;
-
-/// The device is borrowed for a VFIO write callback on the current thread.
-pub struct Write;
-
-/// The device is borrowed for a VFIO mmap callback on the current thread.
-pub struct Mmap;
-
-/// The device is borrowed for a VFIO region-info callback on the current thread.
-pub struct GetRegionInfo;
-
-mod private {
-    pub trait Sealed {}
-
-    impl Sealed for super::Normal {}
-    impl Sealed for super::Open {}
-    impl Sealed for super::Write {}
-    impl Sealed for super::Mmap {}
-    impl Sealed for super::Ioctl {}
-    impl Sealed for super::Read {}
-    impl Sealed for super::GetRegionInfo {}
-}
-
-impl DeviceContext for Normal {}
-impl DeviceContext for Open {}
-impl DeviceContext for Write {}
-impl DeviceContext for Mmap {}
-impl DeviceContext for Ioctl {}
-impl DeviceContext for Read {}
-impl DeviceContext for GetRegionInfo {}
-
 /// Connect a PCI driver's private data to its VFIO registration.
 ///
 /// # Safety
@@ -399,20 +365,22 @@ pub fn region_end(&self) -> Result<u64> {
 /// - Callback contexts are only borrowed for the corresponding callback and thread.
 /// - `core_device` was initialized by `vfio_pci_core_init_dev()`.
 /// - The VFIO device refcount owns the allocation lifetime.
-/// - `reg_data` is dangling without an owner or points to a valid
-///   `<T::RegistrationData as ForLt>::Of<'_>` owned by the enclosing [`Registration`].
-/// - `open_data` is null outside a successful open, otherwise it owns a pinned
-///   `KBox<T::OpenData<'_>>` until last close. Only open/close mutate it; VFIO
-///   prevents I/O callbacks from racing those transitions. The data is dropped
-///   before PCI core close and before registration data is freed.
+/// - `data` follows the VFIO registration and first-open/last-close lifetimes.
+///   Open data is destroyed before PCI core close, and registration data is
+///   released after PCI core unregistration has drained callbacks.
 #[repr(C)]
 pub struct Device<T: Operations, Ctx: DeviceContext = Normal> {
     core_device: Opaque<bindings::vfio_pci_core_device>,
-    reg_data: UnsafeCell<NonNull<<T::RegistrationData as ForLt>::Of<'static>>>,
-    open_data: UnsafeCell<*mut T::OpenData<'static>>,
+    data: CallbackData<T::RegistrationData, OpenDataFamily<T>>,
     _context: PhantomData<(Ctx, NotThreadSafe)>,
 }
 
+struct OpenDataFamily<T>(PhantomData<T>);
+
+impl<T: Operations> ForLt for OpenDataFamily<T> {
+    type Of<'a> = T::OpenData<'a>;
+}
+
 impl<T: Operations> Device<T> {
     /// Allocate a VFIO PCI device for subsequent registration.
     pub fn new(pdev: &pci::Device<device::Core<'_>>) -> Result<ARef<Self>> {
@@ -426,6 +394,7 @@ pub fn new_passthrough(pdev: &pci::Device<device::Core<'_>>) -> Result<ARef<Self
 
     fn allocate(pdev: &pci::Device<device::Core<'_>>, passthrough: bool) -> Result<ARef<Self>> {
         const_assert!(core::mem::offset_of!(Self, core_device) == 0);
+        const_assert!(core::mem::offset_of!(bindings::vfio_pci_core_device, vdev) == 0);
         let size = Kmalloc::aligned_layout(Layout::new::<Self>()).size();
 
         // SAFETY: The bound PCI device and static ops are valid. The allocation
@@ -437,8 +406,7 @@ fn allocate(pdev: &pci::Device<device::Core<'_>>, passthrough: bool) -> Result<A
 
         // SAFETY: Initialise the Rust field in the newly allocated device.
         unsafe {
-            (&raw mut (*this.as_ptr()).reg_data).write(UnsafeCell::new(NonNull::dangling()));
-            (&raw mut (*this.as_ptr()).open_data).write(UnsafeCell::new(core::ptr::null_mut()));
+            (&raw mut (*this.as_ptr()).data).write(CallbackData::new());
             (&raw mut (*this.as_ptr())._context).write(PhantomData);
         }
 
@@ -451,50 +419,6 @@ fn allocate(pdev: &pci::Device<device::Core<'_>>, passthrough: bool) -> Result<A
         Ok(unsafe { ARef::from_raw(this) })
     }
 
-    /// Access the registration data through a closure with an HRTB lifetime.
-    ///
-    /// The closure's `for<'a>` bound prevents the caller from smuggling
-    /// references with a concrete short lifetime out of the closure.
-    ///
-    /// # Safety
-    ///
-    /// Registration data must be published and remain valid through this call.
-    unsafe fn registration_data_with<R>(
-        &self,
-        f: impl for<'a> FnOnce(&'a <T::RegistrationData as ForLt>::Of<'a>) -> R,
-    ) -> R {
-        // SAFETY: Registration keeps the allocation and its borrowed resources
-        // alive until unregistration completes. Covariance permits shortening
-        // the erased binding lifetime to this callback's borrow.
-        let reg_data: &<T::RegistrationData as ForLt>::Of<'_> = unsafe {
-            (*self.reg_data.get())
-                .cast::<<T::RegistrationData as ForLt>::Of<'_>>()
-                .as_ref()
-        };
-        f(reg_data)
-    }
-
-    /// # Safety
-    ///
-    /// The caller must be an I/O callback while VFIO keeps the device open.
-    unsafe fn callback_data_with<R>(
-        &self,
-        f: impl for<'borrow, 'data> FnOnce(
-            &'borrow <T::RegistrationData as ForLt>::Of<'data>,
-            Pin<&'borrow T::OpenData<'data>>,
-        ) -> R,
-    ) -> R {
-        // SAFETY: VFIO excludes open/close and unregister while this callback runs.
-        unsafe {
-            self.registration_data_with(|reg_data| {
-                let ptr = (*self.open_data.get()).cast::<T::OpenData<'_>>();
-                // The data lifetime stays independent of the callback borrow,
-                // so borrowed OpenData fields cannot be stored back into OpenData.
-                f(reg_data, Pin::new_unchecked(&*ptr))
-            })
-        }
-    }
-
     /// # Safety
     /// The returned view must only be used during the callback represented by `Ctx`.
     unsafe fn with_context<Ctx: DeviceContext>(&self) -> &Device<T, Ctx> {
@@ -522,7 +446,15 @@ fn core_device(&self) -> *mut bindings::vfio_pci_core_device {
     }
 
     fn vfio_device(&self) -> *mut bindings::vfio_device {
-        self.core_device.get().cast()
+        self.as_ref().as_raw()
+    }
+}
+
+impl<T: Operations, Ctx: DeviceContext> AsRef<super::Device<Ctx>> for Device<T, Ctx> {
+    fn as_ref(&self) -> &super::Device<Ctx> {
+        // SAFETY: The PCI core device embeds an initialized vfio_device. The borrow
+        // covers the same object and preserves the callback context and its lifetime.
+        unsafe { super::Device::from_raw(&raw mut (*self.core_device()).vdev) }
     }
 }
 
@@ -626,13 +558,14 @@ pub fn core_get_region_info(
 // SAFETY: The embedded device reference count owns the VFIO allocation.
 unsafe impl<T: Operations> AlwaysRefCounted for Device<T> {
     fn inc_ref(&self) {
-        // SAFETY: `self` holds a live VFIO device reference.
-        unsafe { bindings::get_device(&raw mut (*self.vfio_device()).device) };
+        self.as_ref().inc_ref();
     }
 
     unsafe fn dec_ref(obj: NonNull<Self>) {
-        // SAFETY: The caller owns the reference being released.
-        unsafe { bindings::put_device(&raw mut (*obj.as_ref().vfio_device()).device) };
+        // SAFETY: The caller owns a reference to this live PCI wrapper.
+        let dev = unsafe { obj.as_ref() }.as_ref();
+        // SAFETY: The embedded VFIO device owns the same allocation and reference.
+        unsafe { super::Device::dec_ref(NonNull::from(dev)) };
     }
 }
 
@@ -648,9 +581,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
 ///
 /// `vdev` must belong to a registered `Device<T>` in VFIO's first-open context.
 /// VFIO must exclude other opens, closes and I/O callbacks for this call.
-unsafe extern "C" fn open_device_cb<T: Operations>(
-    vdev: *mut bindings::vfio_device,
-) -> core::ffi::c_int {
+unsafe extern "C" fn open_device_cb<T: Operations>(vdev: *mut bindings::vfio_device) -> c_int {
     // SAFETY: `vdev` is valid; set by vfio_alloc_device.
     let dev = unsafe { Device::<T>::from_vfio_device(vdev) };
 
@@ -662,18 +593,9 @@ unsafe impl<T: Operations> Sync for Device<T> {}
 
     // SAFETY: Registration data is initialized before the VFIO device is published.
     let result = unsafe {
-        dev.registration_data_with(|rd| {
-            // Allocate before calling the driver so allocation failure cannot follow open.
-            let data = KBox::<T::OpenData<'_>>::new_uninit(GFP_KERNEL)?;
-            let data = data.write_pin_init(T::open_device(dev.with_context::<Open>(), rd))?;
-
-            // SAFETY: Only the owning pointer is moved; the allocation remains pinned.
-            let raw =
-                KBox::into_raw(Pin::into_inner_unchecked(data)).cast::<T::OpenData<'static>>();
-            // SAFETY: First open is exclusive with I/O callbacks and last close.
-            // Lifetimes do not affect layout; the device owns the allocation until close.
-            *dev.open_data.get() = raw;
-            Ok::<(), Error>(())
+        dev.data.registration_data_with(|rd| {
+            dev.data
+                .open(|| T::open_device(dev.with_context::<Open>(), rd))
         })
     };
     match result {
@@ -698,12 +620,9 @@ unsafe impl<T: Operations> Sync for Device<T> {}
 unsafe extern "C" fn close_device_cb<T: Operations>(vdev: *mut bindings::vfio_device) {
     // SAFETY: `vdev` is valid.
     let dev = unsafe { Device::<T>::from_vfio_device(vdev) };
-    // SAFETY: Last close is exclusive with I/O callbacks and the next first open.
-    let raw = unsafe { core::mem::replace(&mut *dev.open_data.get(), core::ptr::null_mut()) };
-    // Run the driver's destructor while registration data and PCI resources are valid.
-    // SAFETY: Successful open transferred this allocation with `KBox::into_raw`.
-    // Last close takes ownership exactly once and drops it without moving the pointee.
-    unsafe { drop(KBox::from_raw(raw)) };
+    // SAFETY: Last close drains I/O callbacks and excludes the next first open.
+    // Registration data and PCI resources remain valid while open data is destroyed.
+    unsafe { dev.data.close() };
     // SAFETY: Matches the enable in open_device_cb.
     unsafe { bindings::vfio_pci_core_close_device(vdev) };
 }
@@ -721,7 +640,8 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     let dev = unsafe { Device::<T>::from_vfio_device(vdev) };
     // SAFETY: VFIO invokes this callback for an open device with valid arguments.
     match unsafe {
-        dev.callback_data_with(|rd, od| T::ioctl(dev.with_context::<Ioctl>(), rd, od, cmd, arg))
+        dev.data
+            .callback_data_with(|rd, od| T::ioctl(dev.with_context::<Ioctl>(), rd, od, cmd, arg))
     } {
         Ok(v) => v,
         Err(e) => e.to_errno() as isize,
@@ -752,7 +672,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     };
     // SAFETY: VFIO invokes this callback for an open device with valid arguments.
     match unsafe {
-        dev.callback_data_with(|rd, od| {
+        dev.data.callback_data_with(|rd, od| {
             T::read(dev.with_context::<Read>(), rd, od, &mut ubuf, &mut pos)
         })
     } {
@@ -771,7 +691,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     vdev: *mut bindings::vfio_device,
     info: *mut bindings::vfio_region_info,
     caps: *mut bindings::vfio_info_cap,
-) -> core::ffi::c_int {
+) -> c_int {
     // SAFETY: `vdev`, `info`, and `caps` are valid kernel pointers provided
     // by the VFIO core.
     let dev = unsafe { Device::<T>::from_vfio_device(vdev) };
@@ -783,7 +703,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     };
     // SAFETY: VFIO invokes this callback for an open device with valid arguments.
     match unsafe {
-        dev.callback_data_with(|rd, od| {
+        dev.data.callback_data_with(|rd, od| {
             T::get_region_info(dev.with_context::<GetRegionInfo>(), rd, od, info, &mut caps)
         })
     } {
@@ -815,7 +735,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     };
     // SAFETY: This callback runs while VFIO keeps both callback data allocations alive.
     match unsafe {
-        dev.callback_data_with(|rd, od| {
+        dev.data.callback_data_with(|rd, od| {
             T::write(dev.with_context::<Write>(), rd, od, &mut ubuf, &mut pos)
         })
     } {
@@ -832,7 +752,7 @@ unsafe impl<T: Operations> Sync for Device<T> {}
 unsafe extern "C" fn mmap_cb<T: Operations>(
     vdev: *mut bindings::vfio_device,
     vma: *mut bindings::vm_area_struct,
-) -> core::ffi::c_int {
+) -> c_int {
     // SAFETY: VFIO supplies an open device for this mmap callback.
     let dev = unsafe { Device::<T>::from_vfio_device(vdev) };
     let mut mapping = Mapping {
@@ -842,7 +762,8 @@ unsafe impl<T: Operations> Sync for Device<T> {}
     };
     // SAFETY: This callback runs while VFIO keeps both callback data allocations alive.
     match unsafe {
-        dev.callback_data_with(|rd, od| T::mmap(dev.with_context::<Mmap>(), rd, od, &mut mapping))
+        dev.data
+            .callback_data_with(|rd, od| T::mmap(dev.with_context::<Mmap>(), rd, od, &mut mapping))
     } {
         Ok(()) => 0,
         Err(e) => e.to_errno(),
@@ -878,7 +799,11 @@ unsafe fn registration<'a>(
             return;
         };
         // SAFETY: The registration owns callback data throughout bind and unbind.
-        unsafe { reg.dev.registration_data_with(D::Operations::reset_prepare) };
+        unsafe {
+            reg.dev
+                .data
+                .registration_data_with(D::Operations::reset_prepare)
+        };
     }
 
     unsafe extern "C" fn reset_done(pdev: *mut bindings::pci_dev) {
@@ -887,7 +812,11 @@ unsafe fn registration<'a>(
             return;
         };
         // SAFETY: The registration owns callback data throughout bind and unbind.
-        let result = unsafe { reg.dev.registration_data_with(D::Operations::reset_done) };
+        let result = unsafe {
+            reg.dev
+                .data
+                .registration_data_with(D::Operations::reset_done)
+        };
         if let Err(error) = result {
             // SAFETY: The PCI callback keeps its embedded device alive.
             let dev = unsafe { device::Device::<device::Normal>::from_raw(&raw mut (*pdev).dev) };
@@ -1062,7 +991,7 @@ pub unsafe fn new(
             NonNull::from(Pin::get_ref(reg_data.as_ref())).cast();
 
         // SAFETY: No concurrent registration or callbacks; publish before registering.
-        unsafe { *dev.reg_data.get() = ptr };
+        unsafe { *dev.data.reg_data.get() = ptr };
 
         Ok(Self {
             dev: dev.into(),
@@ -1104,6 +1033,6 @@ impl<T: Operations> Drop for Registration<'_, T> {
     fn drop(&mut self) {
         // SAFETY: The adapter unregisters before removing private data. Failed or unattempted
         // registration also leaves no callbacks behind.
-        unsafe { *self.dev.reg_data.get() = NonNull::dangling() };
+        unsafe { *self.dev.data.reg_data.get() = NonNull::dangling() };
     }
 }
-- 
2.53.0