[PATCH 6/7] gpu: nova-core: gsp: support large RPCs via continuation record

[Eliot Courtney <ecourtney@nvidia.com>]

Splits large RPCs if necessary and sends the remaining parts using
continuation records. RPCs that do not need continuation records
continue to write directly into the command buffer. Ones that do write
into a staging buffer first, so there is one copy.

Continuation record for receive is not necessary to support at the
moment because those replies do not need to be read and are currently
drained by retrying `receive_msg` on ERANGE.

Signed-off-by: Eliot Courtney <ecourtney@nvidia.com>
---
 drivers/gpu/nova-core/gsp/cmdq.rs     |  47 ++++++++++++-
 drivers/gpu/nova-core/gsp/commands.rs | 124 ++++++++++++++++++++++++++++++++++
 drivers/gpu/nova-core/gsp/fw.rs       |   5 ++
 3 files changed, 173 insertions(+), 3 deletions(-)

diff --git a/drivers/gpu/nova-core/gsp/cmdq.rs b/drivers/gpu/nova-core/gsp/cmdq.rs
index 3e9f88eec7cc..c24d813fc587 100644
--- a/drivers/gpu/nova-core/gsp/cmdq.rs
+++ b/drivers/gpu/nova-core/gsp/cmdq.rs
@@ -29,6 +29,10 @@
 use crate::{
     driver::Bar0,
     gsp::{
+        commands::{
+            ContinuationRecord,
+            WrappingCommand, //
+        },
         fw::{
             GspMsgElement,
             MsgFunction,
@@ -524,7 +528,7 @@ fn command_size<M>(command: &M) -> usize
         size_of::<M::Command>() + command.variable_payload_len()
     }
 
-    /// Sends `command` to the GSP.
+    /// Sends `command` to the GSP, without splitting it.
     ///
     /// # Errors
     ///
@@ -533,13 +537,13 @@ fn command_size<M>(command: &M) -> usize
     ///   written to by its [`CommandToGsp::init_variable_payload`] method.
     ///
     /// Error codes returned by the command initializers are propagated as-is.
-    pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
+    fn send_single_command<M>(&mut self, bar: &Bar0, command: &M) -> Result
     where
         M: CommandToGsp,
         // This allows all error types, including `Infallible`, to be used for `M::InitError`.
         Error: From<M::InitError>,
     {
-        let command_size = Self::command_size(&command);
+        let command_size = Self::command_size(command);
         let dst = self.gsp_mem.allocate_command_with_timeout(command_size)?;
 
         // Extract area for the command itself. The GSP message header and the command header
@@ -590,6 +594,43 @@ pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
         Ok(())
     }
 
+    fn send_continuation_record(&mut self, bar: &Bar0, cont: &ContinuationRecord<'_>) -> Result {
+        self.send_single_command(bar, cont)
+    }
+
+    /// Sends `command` to the GSP.
+    ///
+    /// The command may be split into multiple messages if it is large.
+    ///
+    /// # Errors
+    ///
+    /// - `ETIMEDOUT` if space does not become available within the timeout.
+    /// - `EIO` if the variable payload requested by the command has not been entirely
+    ///   written to by its [`CommandToGsp::init_variable_payload`] method.
+    ///
+    /// Error codes returned by the command initializers are propagated as-is.
+    pub(crate) fn send_command<M>(&mut self, bar: &Bar0, command: M) -> Result
+    where
+        M: CommandToGsp,
+        Error: From<M::InitError>,
+    {
+        let msg_max_size = MSGQ_MSG_SIZE_MAX - size_of::<GspMsgElement>();
+        let mut wrapped = WrappingCommand::new(command, msg_max_size)?;
+
+        self.send_single_command(bar, &wrapped)?;
+
+        while let Some(continuation) = wrapped.next_continuation_record() {
+            dev_dbg!(
+                &self.dev,
+                "GSP RPC: send continuation: size=0x{:x}\n",
+                Self::command_size(&continuation),
+            );
+            self.send_continuation_record(bar, &continuation)?;
+        }
+
+        Ok(())
+    }
+
     /// Wait for a message to become available on the message queue.
     ///
     /// This works purely at the transport layer and does not interpret or validate the message
diff --git a/drivers/gpu/nova-core/gsp/commands.rs b/drivers/gpu/nova-core/gsp/commands.rs
index c8430a076269..99603880d56f 100644
--- a/drivers/gpu/nova-core/gsp/commands.rs
+++ b/drivers/gpu/nova-core/gsp/commands.rs
@@ -242,3 +242,127 @@ pub(crate) fn get_gsp_info(cmdq: &mut Cmdq, bar: &Bar0) -> Result<GetGspStaticIn
         }
     }
 }
+
+#[derive(Zeroable)]
+pub(crate) struct Empty {}
+
+// SAFETY: `Empty` is a zero-sized type with no bytes, therefore it trivially has no uninitialized
+// bytes.
+unsafe impl AsBytes for Empty {}
+
+// SAFETY: `Empty` is a zero-sized type with no bytes, therefore it trivially has no uninitialized
+// bytes.
+unsafe impl FromBytes for Empty {}
+
+/// The `ContinuationRecord` command.
+pub(crate) struct ContinuationRecord<'a> {
+    data: &'a [u8],
+}
+
+impl<'a> ContinuationRecord<'a> {
+    /// Creates a new `ContinuationRecord` command with the given data.
+    pub(crate) fn new(data: &'a [u8]) -> Self {
+        Self { data }
+    }
+}
+
+impl<'a> CommandToGsp for ContinuationRecord<'a> {
+    const FUNCTION: MsgFunction = MsgFunction::ContinuationRecord;
+    type Command = Empty;
+    type InitError = Infallible;
+
+    fn init(&self) -> impl Init<Self::Command, Self::InitError> {
+        Empty::init_zeroed()
+    }
+
+    fn variable_payload_len(&self) -> usize {
+        self.data.len()
+    }
+
+    fn init_variable_payload(
+        &self,
+        dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
+    ) -> Result {
+        dst.write_all(self.data)
+    }
+}
+
+/// Wrapper that splits a command across continuation records if needed.
+pub(crate) struct WrappingCommand<C: CommandToGsp> {
+    inner: C,
+    offset: usize,
+    max_size: usize,
+    staging: KVVec<u8>,
+}
+
+impl<C: CommandToGsp> WrappingCommand<C>
+where
+    Error: From<C::InitError>,
+{
+    /// Creates a new `WrappingCommand` that wraps `inner`, splitting it into
+    /// multiple messages if its size exceeds `max_size`.
+    pub(crate) fn new(inner: C, max_size: usize) -> Result<Self> {
+        let payload_len = inner.variable_payload_len();
+        let command_size = size_of::<C::Command>() + payload_len;
+        let (offset, staging) = if command_size > max_size {
+            let mut staging = KVVec::<u8>::from_elem(0u8, payload_len, GFP_KERNEL)?;
+            let mut sbuffer = SBufferIter::new_writer([staging.as_mut_slice(), &mut []]);
+            inner.init_variable_payload(&mut sbuffer)?;
+            if !sbuffer.is_empty() {
+                return Err(EIO);
+            }
+            drop(sbuffer);
+
+            (max_size - size_of::<C::Command>(), staging)
+        } else {
+            (0, KVVec::new())
+        };
+        Ok(Self {
+            inner,
+            offset,
+            max_size,
+            staging,
+        })
+    }
+
+    pub(crate) fn next_continuation_record(&mut self) -> Option<ContinuationRecord<'_>> {
+        let remaining = self.staging.len() - self.offset;
+        if remaining > 0 {
+            let chunk_size = remaining.min(self.max_size);
+            let record = ContinuationRecord::new(
+                &self.staging.as_slice()[self.offset..(self.offset + chunk_size)],
+            );
+            self.offset += chunk_size;
+            Some(record)
+        } else {
+            None
+        }
+    }
+}
+
+impl<C: CommandToGsp> CommandToGsp for WrappingCommand<C> {
+    const FUNCTION: MsgFunction = C::FUNCTION;
+    type Command = C::Command;
+    type InitError = C::InitError;
+
+    fn init(&self) -> impl Init<Self::Command, Self::InitError> {
+        self.inner.init()
+    }
+
+    fn variable_payload_len(&self) -> usize {
+        self.inner
+            .variable_payload_len()
+            .min(self.max_size - size_of::<C::Command>())
+    }
+
+    fn init_variable_payload(
+        &self,
+        dst: &mut SBufferIter<core::array::IntoIter<&mut [u8], 2>>,
+    ) -> Result {
+        if self.staging.is_empty() {
+            self.inner.init_variable_payload(dst)
+        } else {
+            dst.write_all(&self.staging.as_slice()[..self.variable_payload_len()])
+        }
+    }
+}
diff --git a/drivers/gpu/nova-core/gsp/fw.rs b/drivers/gpu/nova-core/gsp/fw.rs
index 5b96f5e622a4..5ceb070aa0d7 100644
--- a/drivers/gpu/nova-core/gsp/fw.rs
+++ b/drivers/gpu/nova-core/gsp/fw.rs
@@ -199,6 +199,7 @@ pub(crate) enum MsgFunction {
     AllocObject = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT,
     AllocRoot = bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT,
     BindCtxDma = bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA,
+    ContinuationRecord = bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD,
     Free = bindings::NV_VGPU_MSG_FUNCTION_FREE,
     GetGspStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO,
     GetStaticInfo = bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO,
@@ -234,6 +235,7 @@ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
             MsgFunction::AllocObject => write!(f, "ALLOC_OBJECT"),
             MsgFunction::AllocRoot => write!(f, "ALLOC_ROOT"),
             MsgFunction::BindCtxDma => write!(f, "BIND_CTX_DMA"),
+            MsgFunction::ContinuationRecord => write!(f, "CONTINUATION_RECORD"),
             MsgFunction::Free => write!(f, "FREE"),
             MsgFunction::GetGspStaticInfo => write!(f, "GET_GSP_STATIC_INFO"),
             MsgFunction::GetStaticInfo => write!(f, "GET_STATIC_INFO"),
@@ -273,6 +275,9 @@ fn try_from(value: u32) -> Result<MsgFunction> {
             bindings::NV_VGPU_MSG_FUNCTION_ALLOC_OBJECT => Ok(MsgFunction::AllocObject),
             bindings::NV_VGPU_MSG_FUNCTION_ALLOC_ROOT => Ok(MsgFunction::AllocRoot),
             bindings::NV_VGPU_MSG_FUNCTION_BIND_CTX_DMA => Ok(MsgFunction::BindCtxDma),
+            bindings::NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD => {
+                Ok(MsgFunction::ContinuationRecord)
+            }
             bindings::NV_VGPU_MSG_FUNCTION_FREE => Ok(MsgFunction::Free),
             bindings::NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO => Ok(MsgFunction::GetGspStaticInfo),
             bindings::NV_VGPU_MSG_FUNCTION_GET_STATIC_INFO => Ok(MsgFunction::GetStaticInfo),

-- 
2.53.0