[PATCH v3 14/33] gpu: nova-core: add the falcon DMA and suspend helpers for r000 boot

[John Hubbard <jhubbard@nvidia.com>]

The r000 boot protocol has two load-and-execute events. In each, GSP-RM
names an image that it has placed in the framebuffer, and the driver
copies the image into the GSP falcon's memory by DMA and runs it. The
event supplies the image's framebuffer address, and the copy goes
through a context DMA slot of the falcon's framebuffer interface. GSP-RM
also reports that the GSP's RISC-V core has suspended through a bit in
the falcon's MAILBOX0 register, rather than through the halted bit in
CPUCTL, and the driver has to wait for that bit before it resets the
falcon.

Nova-core's falcon DMA copied only from a DMA object that the driver had
allocated, so there was no way to copy an image from an address that an
event supplied. The shutdown path read the suspend bit inline, so there
was no wait for it that a boot event handler could share.

Add a DMA transfer that copies from a caller-supplied source address
through a caller-supplied context DMA slot. Add a check of the suspend
bit and a wait built on it, and make the shutdown path use the check in
place of its inline read.

Assisted-by: LLM
Reviewed-by: Timur Tabi <ttabi@nvidia.com>
Reviewed-by: Zhi Wang <zhiw@nvidia.com>
Signed-off-by: John Hubbard <jhubbard@nvidia.com>
---
 drivers/gpu/nova-core/falcon.rs   | 150 +++++++++++++++++++++++++++++-
 drivers/gpu/nova-core/gsp/boot.rs |   6 +-
 drivers/gpu/nova-core/regs.rs     |   2 +-
 3 files changed, 151 insertions(+), 7 deletions(-)

diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 04e35cbcb6f0..2a0fe86153aa 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -5,6 +5,7 @@
 use hal::FalconHal;
 
 use kernel::{
+    bits,
     device,
     dma::{
         Coherent,
@@ -20,7 +21,7 @@
     },
     prelude::*,
     sizes::SZ_4K,
-    time::Delta,
+    time::Delta, //
 };
 
 use crate::{
@@ -43,6 +44,9 @@
 /// Alignment (in bytes) of falcon memory blocks.
 pub(crate) const MEM_BLOCK_ALIGNMENT: usize = 256;
 
+/// DMEM virtual address value that means "no virtual address assigned".
+const FLCN_DMEM_VA_INVALID: u32 = 0xffff_ffff;
+
 bounded_enum! {
     /// Revision number of a falcon core, used in the [`crate::regs::NV_PFALCON_FALCON_HWCFG1`]
     /// register.
@@ -132,12 +136,36 @@ pub(crate) enum FalconMem {
     /// Secure Instruction Memory.
     ImemSecure,
     /// Non-Secure Instruction Memory.
-    #[expect(unused)]
+    #[expect(dead_code)]
     ImemNonSecure,
     /// Data Memory.
     Dmem,
 }
 
+/// Where a raw falcon DMA transfer reads its image from, relative to the DMA base address.
+#[derive(Copy, Clone)]
+pub(crate) enum FalconDmaSrcOffset {
+    /// The image starts this many bytes past the DMA base address.
+    Offset(u32),
+    /// Virtual address of the image in the target memory. The DMA engine tags each loaded block
+    /// with this value and adds the tag to the base address, so the base address must be the
+    /// source address minus this value.
+    Va(u32),
+}
+
+impl FalconDmaSrcOffset {
+    /// Returns the source offset of a DMEM image at virtual address `dmem_va`, or the start of the
+    /// source when `dmem_va` is `FLCN_DMEM_VA_INVALID`.
+    #[expect(dead_code)]
+    pub(crate) fn from_dmem_va(dmem_va: u32) -> Self {
+        if dmem_va == FLCN_DMEM_VA_INVALID {
+            Self::Offset(0)
+        } else {
+            Self::Va(dmem_va)
+        }
+    }
+}
+
 bounded_enum! {
     /// Defines the Framebuffer Interface (FBIF) aperture type.
     /// This determines the memory type for external memory access during a DMA transfer, which is
@@ -600,6 +628,98 @@ fn dma_wr(
         Ok(())
     }
 
+    /// Transfers `len` bytes from `src_addr` into this falcon's `target_mem`.
+    ///
+    /// `src_addr` is a GPU physical address reached through the FBIF aperture, so the caller must
+    /// program `NV_PFALCON_FBIF_TRANSCFG` for `ctx_dma` before calling this.
+    ///
+    /// # Errors
+    ///
+    /// - `EINVAL` if `ctx_dma` is not a context DMA slot that the falcon has, or if `src_addr` is
+    ///   not 256-byte aligned.
+    /// - `ERANGE` if `src_addr` does not fit the `DMATRFBASE` register pair.
+    /// - `EOVERFLOW` if a per-block source or destination offset exceeds `u32`.
+    #[expect(dead_code)]
+    pub(crate) fn raw_dma_transfer(
+        &self,
+        ctx_dma: u32,
+        src_addr: u64,
+        target_mem: FalconMem,
+        src: FalconDmaSrcOffset,
+        dst_offset: u32,
+        len: u32,
+    ) -> Result {
+        const DMA_LEN: u32 = num::usize_into_u32::<{ MEM_BLOCK_ALIGNMENT }>();
+
+        if src_addr % u64::from(DMA_LEN) > 0 {
+            dev_err!(
+                self.dev,
+                "raw DMA: source address {:#x} not 256B-aligned\n",
+                src_addr
+            );
+            return Err(EINVAL);
+        }
+
+        if src_addr >> 40 > u64::from(regs::NV_PFALCON_FALCON_DMATRFBASE1::BASE_MASK) {
+            dev_err!(
+                self.dev,
+                "raw DMA: source address {:#x} does not fit DMATRFBASE\n",
+                src_addr
+            );
+            return Err(ERANGE);
+        }
+
+        // An IMEM block is always tagged with its source offset. A DMEM block is tagged only when
+        // `SET_DMTAG` is set, so a DMEM virtual address sets `SET_DMTAG`.
+        let (src_offset, set_dmtag) = match src {
+            FalconDmaSrcOffset::Offset(offset) => (offset, false),
+            FalconDmaSrcOffset::Va(va) => (va, target_mem == FalconMem::Dmem),
+        };
+
+        let num_transfers = len.div_ceil(DMA_LEN);
+
+        self.pfalcon
+            .write_reg(regs::NV_PFALCON_FALCON_DMATRFBASE::zeroed().with_base(
+                // CAST: this drops the upper bits on purpose. They are written to
+                // `NV_PFALCON_FALCON_DMATRFBASE1` next.
+                (src_addr >> 8) as u32,
+            ));
+        self.pfalcon.write_reg(
+            regs::NV_PFALCON_FALCON_DMATRFBASE1::zeroed().try_with_base(src_addr >> 40)?,
+        );
+
+        // The `CTXDMA` field holds exactly the indices of the falcon's `TRANSCFG` slots, so the
+        // field's range is the bound on the slot.
+        let cmd = regs::NV_PFALCON_FALCON_DMATRFCMD::zeroed()
+            .with_size(DmaTrfCmdSize::Size256B)
+            .try_with_ctxdma(ctx_dma)
+            .map_err(|_| EINVAL)?
+            .with_falcon_mem(target_mem)
+            .with_set_dmtag(set_dmtag);
+
+        for pos in (0..num_transfers).map(|i| i * DMA_LEN) {
+            self.pfalcon.write_reg(
+                regs::NV_PFALCON_FALCON_DMATRFMOFFS::zeroed()
+                    .try_with_offs(dst_offset.checked_add(pos).ok_or(EOVERFLOW)?)?,
+            );
+            self.pfalcon.write_reg(
+                regs::NV_PFALCON_FALCON_DMATRFFBOFFS::zeroed()
+                    .with_offs(src_offset.checked_add(pos).ok_or(EOVERFLOW)?),
+            );
+
+            self.pfalcon.write_reg(cmd);
+
+            read_poll_timeout(
+                || Ok(self.pfalcon.read(regs::NV_PFALCON_FALCON_DMATRFCMD)),
+                |r| r.idle(),
+                Delta::ZERO,
+                Delta::from_secs(2),
+            )?;
+        }
+
+        Ok(())
+    }
+
     /// Perform a DMA load into `IMEM` and `DMEM` of `fw`, and prepare the falcon to run it.
     fn dma_load<F: FalconFirmware<Target = E> + FalconDmaLoadable>(&self, fw: &F) -> Result {
         // DMA object with firmware content as the source of the DMA engine.
@@ -653,6 +773,32 @@ pub(crate) fn wait_till_halted(&self) -> Result<()> {
         Ok(())
     }
 
+    /// Returns `true` if the RISC-V core has suspended.
+    pub(crate) fn is_processor_suspended(&self) -> bool {
+        const INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31);
+
+        self.read_mailbox0() & INTERRUPT_PROCESSOR_SUSPENDED != 0
+    }
+
+    /// Waits until the RISC-V core has suspended.
+    ///
+    /// The caller must write `MAILBOX0` before starting the core, or this returns as soon as it
+    /// reads the previous suspend.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if the core has not suspended within two seconds.
+    #[expect(dead_code)]
+    pub(crate) fn wait_for_processor_suspend(&self) -> Result {
+        read_poll_timeout(
+            || Ok(self.is_processor_suspended()),
+            |suspended| *suspended,
+            Delta::ZERO,
+            Delta::from_secs(2),
+        )
+        .map(|_| ())
+    }
+
     /// Start the falcon CPU.
     pub(crate) fn start(&self) -> Result<()> {
         match self.pfalcon.read(regs::NV_PFALCON_FALCON_CPUCTL).alias_en() {
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index 4fb1b69ac9d5..8518248c9732 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -2,7 +2,6 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 use kernel::{
-    bits,
     io::poll::read_poll_timeout,
     prelude::*,
     time::Delta,
@@ -94,10 +93,9 @@ fn shutdown_gsp(
         cmdq.send_command(commands::UnloadingGuestDriver::new(mode))?;
 
         // Wait until GSP signals it is suspended.
-        const LIBOS_INTERRUPT_PROCESSOR_SUSPENDED: u32 = bits::bit_u32(31);
         read_poll_timeout(
-            || Ok(gsp_falcon.read_mailbox0()),
-            |&mb0| mb0 & LIBOS_INTERRUPT_PROCESSOR_SUSPENDED != 0,
+            || Ok(gsp_falcon.is_processor_suspended()),
+            |suspended| *suspended,
             Delta::from_millis(10),
             Delta::from_secs(5),
         )
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index 69dd6526e469..feb37de69be5 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -203,7 +203,7 @@ pub(crate) fn usable_fb_size(self) -> u64 {
     }
 
     pub(crate) NV_PFALCON_FALCON_DMATRFCMD(u32) @ 0x00000118 {
-        16:16   set_dmtag;
+        16:16   set_dmtag => bool;
         14:12   ctxdma;
         10:8    size ?=> DmaTrfCmdSize;
         5:5     is_write => bool;
-- 
2.55.0