[PATCH v3 15/33] gpu: nova-core: add the r000 load-and-execute HS binary handler

[John Hubbard <jhubbard@nvidia.com>]

On GA102 and later, GSP-RM sends the driver a load-and-execute event
during boot that requests the driver to run a Heavy-Secured (HS) binary
on the GSP falcon. GSP-RM has placed the binary in the framebuffer, and
the event carries the binary's addresses. Once the binary has halted,
SEC2 restarts GSP-RM, in what Open RM calls the core resume.

Add the handler and the core resume. The handler copies the binary into
the falcon through an FBIF (framebuffer interface) aperture that it
programs for the load, starts the binary and waits for it to halt.

Put the falcons, the device and the boot parameters that the handler and
the core resume share into one context struct.

Assisted-by: LLM
Reviewed-by: Timur Tabi <ttabi@nvidia.com>
Signed-off-by: John Hubbard <jhubbard@nvidia.com>
---
 drivers/gpu/nova-core/falcon.rs     |  51 +++++-
 drivers/gpu/nova-core/falcon/gsp.rs |  10 +-
 drivers/gpu/nova-core/gsp/boot.rs   | 257 +++++++++++++++++++++++++++-
 drivers/gpu/nova-core/regs.rs       |   2 +
 drivers/gpu/nova-core/sbuffer.rs    |   1 -
 5 files changed, 311 insertions(+), 10 deletions(-)

diff --git a/drivers/gpu/nova-core/falcon.rs b/drivers/gpu/nova-core/falcon.rs
index 2a0fe86153aa..8ad28dce955e 100644
--- a/drivers/gpu/nova-core/falcon.rs
+++ b/drivers/gpu/nova-core/falcon.rs
@@ -44,6 +44,10 @@
 /// Alignment (in bytes) of falcon memory blocks.
 pub(crate) const MEM_BLOCK_ALIGNMENT: usize = 256;
 
+/// `MAILBOX0` value that means "the falcon binary has not started". A binary that runs replaces
+/// it with its own status.
+pub(crate) const FLCN_ERR_BINARY_NOT_STARTED: u32 = 0xfe;
+
 /// DMEM virtual address value that means "no virtual address assigned".
 const FLCN_DMEM_VA_INVALID: u32 = 0xffff_ffff;
 
@@ -156,7 +160,6 @@ pub(crate) enum FalconDmaSrcOffset {
 impl FalconDmaSrcOffset {
     /// Returns the source offset of a DMEM image at virtual address `dmem_va`, or the start of the
     /// source when `dmem_va` is `FLCN_DMEM_VA_INVALID`.
-    #[expect(dead_code)]
     pub(crate) fn from_dmem_va(dmem_va: u32) -> Self {
         if dmem_va == FLCN_DMEM_VA_INVALID {
             Self::Offset(0)
@@ -192,6 +195,17 @@ pub(crate) enum FalconFbifMemType with From<Bounded<u32, 1>> {
     }
 }
 
+bounded_enum! {
+    /// Engine ID that the falcon's framebuffer interface (FBIF) tags a DMA transfer with.
+    #[derive(Debug, Copy, Clone)]
+    pub(crate) enum FalconFbifEngineIdFlag with From<Bounded<u32, 1>> {
+        /// The BAR2 engine ID of PCI function 0.
+        Bar2Fn0 = 0,
+        /// The falcon's own engine ID.
+        Own = 1,
+    }
+}
+
 const PFALCON_REGION_SIZE: usize = SZ_4K;
 const PFALCON2_REGION_SIZE: usize = SZ_4K;
 
@@ -394,7 +408,7 @@ pub(crate) struct Falcon<'a, E: FalconEngine> {
     bar: Bar0<'a>,
     // TODO: make private
     pub(crate) pfalcon: Mmio<'a, PFalconRegisters>,
-    pfalcon2: Mmio<'a, PFalcon2Registers>,
+    pub(crate) pfalcon2: Mmio<'a, PFalcon2Registers>,
 }
 
 impl<'a, E: FalconEngine + 'static> Falcon<'a, E> {
@@ -628,6 +642,37 @@ fn dma_wr(
         Ok(())
     }
 
+    /// Programs FBIF context DMA slot `ctx_dma` with the value that `configure` returns, runs `f`,
+    /// and restores the slot once `f` has returned `Ok`.
+    ///
+    /// The slot keeps the programmed value if `f` fails, since a falcon that `f` started and that
+    /// has not halted may still be reading through the aperture.
+    ///
+    /// # Errors
+    ///
+    /// - `EINVAL` if `ctx_dma` is not a context DMA slot that the falcon has.
+    ///
+    /// Errors from `f` are propagated as-is.
+    pub(crate) fn with_fbif_transcfg<R>(
+        &self,
+        ctx_dma: u32,
+        configure: impl FnOnce(regs::NV_PFALCON_FBIF_TRANSCFG) -> regs::NV_PFALCON_FBIF_TRANSCFG,
+        f: impl FnOnce() -> Result<R>,
+    ) -> Result<R> {
+        // The location type is not `Copy`, so each register access builds its own.
+        let transcfg =
+            || regs::NV_PFALCON_FBIF_TRANSCFG::try_at(usize::from_safe_cast(ctx_dma)).ok_or(EINVAL);
+
+        let saved = self.pfalcon.read(transcfg()?);
+        self.pfalcon.update(transcfg()?, configure);
+
+        let result = f()?;
+
+        self.pfalcon.update(transcfg()?, |_| saved);
+
+        Ok(result)
+    }
+
     /// Transfers `len` bytes from `src_addr` into this falcon's `target_mem`.
     ///
     /// `src_addr` is a GPU physical address reached through the FBIF aperture, so the caller must
@@ -639,7 +684,6 @@ fn dma_wr(
     ///   not 256-byte aligned.
     /// - `ERANGE` if `src_addr` does not fit the `DMATRFBASE` register pair.
     /// - `EOVERFLOW` if a per-block source or destination offset exceeds `u32`.
-    #[expect(dead_code)]
     pub(crate) fn raw_dma_transfer(
         &self,
         ctx_dma: u32,
@@ -788,7 +832,6 @@ pub(crate) fn is_processor_suspended(&self) -> bool {
     /// # Errors
     ///
     /// - `ETIMEDOUT` if the core has not suspended within two seconds.
-    #[expect(dead_code)]
     pub(crate) fn wait_for_processor_suspend(&self) -> Result {
         read_poll_timeout(
             || Ok(self.is_processor_suspended()),
diff --git a/drivers/gpu/nova-core/falcon/gsp.rs b/drivers/gpu/nova-core/falcon/gsp.rs
index dfa08bc6867c..70f7d55f9b59 100644
--- a/drivers/gpu/nova-core/falcon/gsp.rs
+++ b/drivers/gpu/nova-core/falcon/gsp.rs
@@ -115,15 +115,19 @@ pub(crate) fn retrigger_intr(bar: Bar0<'_>, chipset: Chipset) {
 }
 
 impl<'a> Falcon<'a, Gsp> {
-    /// Checks if GSP reload/resume has completed during the boot process.
-    pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result<bool> {
+    /// Waits until the Boot Sequence Interface (BSI) reports that the GSP reload has completed.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if the reload has not completed within `timeout`.
+    pub(crate) fn check_reload_completed(&self, timeout: Delta) -> Result {
         read_poll_timeout(
             || Ok(self.bar.read(regs::NV_PGC6_BSI_SECURE_SCRATCH_14)),
             |val| val.boot_stage_3_handoff(),
             Delta::ZERO,
             timeout,
         )
-        .map(|_| true)
+        .map(|_| ())
     }
 
     /// Returns whether the RISC-V branch privilege lockdown bit is set.
diff --git a/drivers/gpu/nova-core/gsp/boot.rs b/drivers/gpu/nova-core/gsp/boot.rs
index 8518248c9732..3cfb21964250 100644
--- a/drivers/gpu/nova-core/gsp/boot.rs
+++ b/drivers/gpu/nova-core/gsp/boot.rs
@@ -2,24 +2,230 @@
 // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 use kernel::{
-    io::poll::read_poll_timeout,
+    device,
+    io::{
+        poll::read_poll_timeout,
+        register::Array,
+        Io, //
+    },
     prelude::*,
     time::Delta,
+    transmute::{
+        AsBytes,
+        FromBytes, //
+    },
     types::ScopeGuard, //
 };
 
 use crate::{
     falcon::{
         gsp::Gsp,
-        Falcon, //
+        sec2::Sec2,
+        Falcon,
+        FalconDmaSrcOffset,
+        FalconFbifEngineIdFlag,
+        FalconFbifMemType,
+        FalconFbifTarget,
+        FalconMem,
+        FalconModSelAlgo,
+        FLCN_ERR_BINARY_NOT_STARTED, //
     },
     firmware::gsp::GspFirmware,
     gsp::{
         cmdq::Cmdq,
         commands, //
     },
+    regs,
+    sbuffer::SBufferIter, //
 };
 
+/// The falcons, device and boot parameters that the load-and-execute event handlers share.
+struct LoadExecContext<'a, 'gpu> {
+    gsp_falcon: &'a Falcon<'gpu, Gsp>,
+    sec2_falcon: &'a Falcon<'gpu, Sec2>,
+    dev: &'a device::Device,
+    /// GSP bootloader application version.
+    bootloader_app_version: u32,
+    /// DMA address of the LIBOS init arguments.
+    libos_dma_handle: u64,
+}
+
+impl LoadExecContext<'_, '_> {
+    /// Waits until GSP-RM has suspended the RISC-V core, then resets the GSP falcon and its DMA
+    /// registers.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if the core has not suspended within two seconds.
+    ///
+    /// Errors from the falcon reset are propagated as-is.
+    fn reset_gsp_falcon_after_suspend(&self) -> Result {
+        let Self {
+            gsp_falcon, dev, ..
+        } = *self;
+
+        gsp_falcon.wait_for_processor_suspend().inspect_err(|_| {
+            dev_err!(
+                dev,
+                "Timeout waiting for GSP suspend (mbox0={:#x})\n",
+                gsp_falcon.read_mailbox0()
+            );
+        })?;
+
+        gsp_falcon.reset()?;
+        gsp_falcon.dma_reset();
+
+        Ok(())
+    }
+
+    /// Runs the core resume, in which SEC2 restarts GSP-RM after a load-and-execute binary has
+    /// halted on the GSP falcon.
+    ///
+    /// # Errors
+    ///
+    /// - `EIO` if SEC2 reports a failure, or if the GSP is not running RISC-V afterwards.
+    /// - `ETIMEDOUT` if SEC2 does not complete the reload within two seconds.
+    fn core_resume(&self) -> Result {
+        let Self {
+            gsp_falcon,
+            sec2_falcon,
+            dev,
+            ..
+        } = *self;
+
+        gsp_falcon.reset()?;
+
+        gsp_falcon.write_mailboxes(
+            Some(self.libos_dma_handle as u32),
+            Some((self.libos_dma_handle >> 32) as u32),
+        );
+
+        sec2_falcon.start()?;
+
+        gsp_falcon
+            .check_reload_completed(Delta::from_secs(2))
+            .inspect_err(|_| {
+                let mbox0 = sec2_falcon.read_mailbox0();
+                dev_err!(
+                    dev,
+                    "Timeout waiting for SEC2 to resume GSP-RM (SEC2 mbox0={:#x})\n",
+                    mbox0
+                );
+            })?;
+
+        let sec2_mbox0 = sec2_falcon.read_mailbox0();
+        if sec2_mbox0 != 0 {
+            dev_err!(
+                dev,
+                "SEC2 reported error during core resume: {:#x}\n",
+                sec2_mbox0
+            );
+            return Err(EIO);
+        }
+
+        gsp_falcon.write_os_version(self.bootloader_app_version);
+
+        if !gsp_falcon.is_riscv_active() {
+            dev_err!(dev, "GSP RISC-V not active after core resume\n");
+            return Err(EIO);
+        }
+
+        Ok(())
+    }
+
+    /// Runs a Heavy-Secured (HS) binary on the GSP falcon, as a `GMCAPI_CMD_EXEC_HS_BINARY` event
+    /// requests, and then restarts GSP-RM.
+    ///
+    /// GSP-RM has placed the binary in the framebuffer, and the falcon's boot ROM (BROM) verifies
+    /// the binary's signature before the binary runs.
+    ///
+    /// # Errors
+    ///
+    /// - `EINVAL` if the payload is shorter than the parameter block, or the ucode id does not
+    ///   fit the BROM register field.
+    /// - `ETIMEDOUT` if the RISC-V core does not suspend within two seconds, or the GSP falcon does
+    ///   not halt within two seconds of starting the binary.
+    ///
+    /// Errors from [`Self::core_resume`] are propagated as-is.
+    #[expect(dead_code)]
+    fn handle_load_exec_hs_binary(&self, payload_0: &[u8], payload_1: &[u8]) -> Result {
+        let Self {
+            gsp_falcon, dev, ..
+        } = *self;
+        let params = read_params::<HsBinaryParams>(payload_0, payload_1)?;
+
+        self.reset_gsp_falcon_after_suspend()?;
+
+        gsp_falcon.with_fbif_transcfg(
+            HsBinaryParams::CTX_DMA,
+            |v| {
+                v.with_target(FalconFbifTarget::LocalFb)
+                    .with_mem_type(FalconFbifMemType::Physical)
+                    .with_engine_id_flag(FalconFbifEngineIdFlag::Bar2Fn0)
+            },
+            || {
+                if params.ucode_imem_size > 0 {
+                    gsp_falcon.raw_dma_transfer(
+                        HsBinaryParams::CTX_DMA,
+                        params.imem_phys_addr,
+                        FalconMem::ImemSecure,
+                        FalconDmaSrcOffset::Va(params.ucode_imem_va),
+                        params.ucode_imem_pa,
+                        params.ucode_imem_size,
+                    )?;
+                }
+
+                if params.ucode_dmem_size > 0 {
+                    gsp_falcon.raw_dma_transfer(
+                        HsBinaryParams::CTX_DMA,
+                        params.dmem_phys_addr,
+                        FalconMem::Dmem,
+                        FalconDmaSrcOffset::from_dmem_va(params.ucode_dmem_va),
+                        params.ucode_dmem_pa,
+                        params.ucode_dmem_size,
+                    )?;
+                }
+
+                gsp_falcon.pfalcon2.write(
+                    Array::at(0),
+                    regs::NV_PFALCON2_FALCON_BROM_PARAADDR::zeroed()
+                        .with_value(params.hs_sig_dmem_addr),
+                );
+                gsp_falcon.pfalcon2.write_reg(
+                    regs::NV_PFALCON2_FALCON_BROM_ENGIDMASK::zeroed()
+                        .with_value(params.engine_id_mask),
+                );
+                gsp_falcon.pfalcon2.write_reg(
+                    regs::NV_PFALCON2_FALCON_BROM_CURR_UCODE_ID::zeroed()
+                        .with_ucode_id(u8::try_from(params.ucode_id).map_err(|_| EINVAL)?),
+                );
+                gsp_falcon.pfalcon2.write_reg(
+                    regs::NV_PFALCON2_FALCON_MOD_SEL::zeroed().with_algo(FalconModSelAlgo::Rsa3k),
+                );
+
+                gsp_falcon.pfalcon.write_reg(
+                    regs::NV_PFALCON_FALCON_BOOTVEC::zeroed().with_value(params.ucode_imem_va),
+                );
+
+                let (mbox0, _) = gsp_falcon
+                    .boot(Some(FLCN_ERR_BINARY_NOT_STARTED), None)
+                    .inspect_err(|_| {
+                        dev_err!(
+                            dev,
+                            "Timeout waiting for HS binary to halt (mbox0={:#x})\n",
+                            gsp_falcon.read_mailbox0()
+                        );
+                    })?;
+                dev_dbg!(dev, "HS binary halted with mbox0={:#x}\n", mbox0);
+
+                Ok(())
+            },
+        )?;
+
+        self.core_resume()
+    }
+}
+
 impl<'gsp> super::Gsp<'gsp> {
     /// Attempt to boot the GSP.
     ///
@@ -140,3 +346,50 @@ pub(crate) fn unload(
         res.inspect(|()| dev_info!(dev, "GSP successfully unloaded\n"))
     }
 }
+
+/// Reads the parameter block of type `T` from the start of an event payload, which the ring may
+/// have split in two.
+///
+/// # Errors
+///
+/// - `EINVAL` if the payload is shorter than `T`.
+fn read_params<T: FromBytes + AsBytes + Zeroable>(payload_0: &[u8], payload_1: &[u8]) -> Result<T> {
+    let mut params = T::zeroed();
+
+    SBufferIter::new_reader([payload_0, payload_1]).read_exact(params.as_bytes_mut())?;
+
+    Ok(params)
+}
+
+/// Payload of a `GMCAPI_CMD_EXEC_HS_BINARY` event.
+///
+/// GSP-RM has written the code to `imem_phys_addr` and the data to `dmem_phys_addr` in the
+/// framebuffer before it sends the event.
+#[repr(C)]
+#[derive(Debug, Copy, Clone, Zeroable)]
+struct HsBinaryParams {
+    imem_phys_addr: u64,
+    dmem_phys_addr: u64,
+    _reserved64: [u64; 2],
+    ucode_imem_va: u32,
+    ucode_imem_pa: u32,
+    ucode_imem_size: u32,
+    ucode_dmem_va: u32,
+    ucode_dmem_pa: u32,
+    ucode_dmem_size: u32,
+    hs_sig_dmem_addr: u32,
+    engine_id_mask: u32,
+    ucode_id: u32,
+    _reserved32: [u32; 3],
+}
+
+impl HsBinaryParams {
+    /// Context DMA slot through which the binary is loaded.
+    const CTX_DMA: u32 = 0;
+}
+
+// SAFETY: This struct only contains integer types for which all bit patterns are valid.
+unsafe impl FromBytes for HsBinaryParams {}
+
+// SAFETY: This struct only contains integer types, laid out without padding.
+unsafe impl AsBytes for HsBinaryParams {}
diff --git a/drivers/gpu/nova-core/regs.rs b/drivers/gpu/nova-core/regs.rs
index feb37de69be5..ec8e05dc3351 100644
--- a/drivers/gpu/nova-core/regs.rs
+++ b/drivers/gpu/nova-core/regs.rs
@@ -18,6 +18,7 @@
         DmaTrfCmdSize,
         FalconCoreRev,
         FalconCoreRevSubversion,
+        FalconFbifEngineIdFlag,
         FalconFbifMemType,
         FalconFbifTarget,
         FalconMem,
@@ -286,6 +287,7 @@ pub(crate) fn usable_fb_size(self) -> u64 {
     }
 
     pub(crate) NV_PFALCON_FBIF_TRANSCFG(u32)[8] @ 0x00000600 {
+        16:16   engine_id_flag => FalconFbifEngineIdFlag;
         2:2     mem_type => FalconFbifMemType;
         1:0     target ?=> FalconFbifTarget;
     }
diff --git a/drivers/gpu/nova-core/sbuffer.rs b/drivers/gpu/nova-core/sbuffer.rs
index 3a41d224c77a..b8c01104c255 100644
--- a/drivers/gpu/nova-core/sbuffer.rs
+++ b/drivers/gpu/nova-core/sbuffer.rs
@@ -146,7 +146,6 @@ fn get_slice(&mut self, len: usize) -> Option<&'a [u8]> {
 
     /// Ideally we would implement `Read`, but it is not available in `core`.
     /// So mimic `std::io::Read::read_exact`.
-    #[expect(unused)]
     pub(crate) fn read_exact(&mut self, mut dst: &mut [u8]) -> Result {
         while !dst.is_empty() {
             match self.get_slice(dst.len()) {
-- 
2.55.0