Re: [PATCH v4 06/28] drm/xe/pf: Add helpers for migration data packet allocation / free

[Michal Wajdeczko <michal.wajdeczko@intel.com>]

On 11/5/2025 4:10 PM, Michał Winiarski wrote:
> Now that it's possible to free the packets - connect the restore
> handling logic with the ring.
> The helpers will also be used in upcoming changes that will start
> producing migration data packets.
> 
> Signed-off-by: Michał Winiarski <michal.winiarski@intel.com>
> Reviewed-by: Michal Wajdeczko <michal.wajdeczko@intel.com>
> ---
>  drivers/gpu/drm/xe/Makefile                   |   1 +
>  drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c   |  11 +-
>  drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c |  33 ++++-
>  drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h |   1 +
>  drivers/gpu/drm/xe/xe_sriov_packet.c          | 129 ++++++++++++++++++
>  drivers/gpu/drm/xe/xe_sriov_packet.h          |  30 ++++
>  6 files changed, 202 insertions(+), 3 deletions(-)
>  create mode 100644 drivers/gpu/drm/xe/xe_sriov_packet.c
>  create mode 100644 drivers/gpu/drm/xe/xe_sriov_packet.h
> 
> diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile
> index cb6b30fd8c2d8..e4b273b025d2a 100644
> --- a/drivers/gpu/drm/xe/Makefile
> +++ b/drivers/gpu/drm/xe/Makefile
> @@ -174,6 +174,7 @@ xe-$(CONFIG_PCI_IOV) += \
>  	xe_lmtt_2l.o \
>  	xe_lmtt_ml.o \
>  	xe_pci_sriov.o \
> +	xe_sriov_packet.o \
>  	xe_sriov_pf.o \
>  	xe_sriov_pf_control.o \
>  	xe_sriov_pf_debugfs.o \
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
> index db57f9d9b491f..25937592bc42f 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_control.c
> @@ -18,6 +18,7 @@
>  #include "xe_gt_sriov_printk.h"
>  #include "xe_guc_ct.h"
>  #include "xe_sriov.h"
> +#include "xe_sriov_packet.h"
>  #include "xe_sriov_pf_control.h"
>  #include "xe_sriov_pf_migration.h"
>  #include "xe_sriov_pf_service.h"
> @@ -852,6 +853,8 @@ int xe_gt_sriov_pf_control_resume_vf(struct xe_gt *gt, unsigned int vfid)
>  static void pf_exit_vf_save_wip(struct xe_gt *gt, unsigned int vfid)
>  {
>  	if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WIP)) {
> +		xe_gt_sriov_pf_migration_ring_free(gt, vfid);
> +
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_PROCESS_DATA);
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_WAIT_DATA);
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_SAVE_DATA_DONE);
> @@ -1074,6 +1077,8 @@ int xe_gt_sriov_pf_control_finish_save_vf(struct xe_gt *gt, unsigned int vfid)
>  static void pf_exit_vf_restore_wip(struct xe_gt *gt, unsigned int vfid)
>  {
>  	if (pf_exit_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WIP)) {
> +		xe_gt_sriov_pf_migration_ring_free(gt, vfid);
> +
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_PROCESS_DATA);
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_WAIT_DATA);
>  		pf_escape_vf_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_DATA_DONE);
> @@ -1108,6 +1113,8 @@ static int pf_handle_vf_restore_data(struct xe_gt *gt, unsigned int vfid)
>  
>  	xe_gt_sriov_notice(gt, "Skipping VF%u unknown data type: %d\n", vfid, data->type);
>  
> +	xe_sriov_packet_free(data);
> +
>  	return 0;
>  }
>  
> @@ -1201,8 +1208,10 @@ int xe_gt_sriov_pf_control_restore_data_done(struct xe_gt *gt, unsigned int vfid
>   */
>  int xe_gt_sriov_pf_control_process_restore_data(struct xe_gt *gt, unsigned int vfid)
>  {
> -	if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED))
> +	if (!pf_expect_vf_not_state(gt, vfid, XE_GT_SRIOV_STATE_RESTORE_FAILED)) {
> +		xe_gt_sriov_pf_migration_ring_free(gt, vfid);
>  		return -EIO;
> +	}
>  
>  	pf_exit_vf_restore_wait_data(gt, vfid);
>  
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
> index d9ca9ea737dcc..c492677db28d9 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c
> @@ -14,6 +14,7 @@
>  #include "xe_guc.h"
>  #include "xe_guc_ct.h"
>  #include "xe_sriov.h"
> +#include "xe_sriov_packet.h"
>  #include "xe_sriov_pf_migration.h"
>  
>  #define XE_GT_SRIOV_PF_MIGRATION_RING_SIZE 5
> @@ -418,6 +419,25 @@ bool xe_gt_sriov_pf_migration_ring_full(struct xe_gt *gt, unsigned int vfid)
>  	return ptr_ring_full(&pf_pick_gt_migration(gt, vfid)->ring);
>  }
>  
> +/**
> + * xe_gt_sriov_pf_migration_ring_free() - Consume and free all data in migration ring
> + * @gt: the &xe_gt
> + * @vfid: the VF identifier
> + */
> +void xe_gt_sriov_pf_migration_ring_free(struct xe_gt *gt, unsigned int vfid)
> +{
> +	struct xe_gt_sriov_migration_data *migration = pf_pick_gt_migration(gt, vfid);
> +	struct xe_sriov_packet *data;
> +
> +	if (ptr_ring_empty(&migration->ring))
> +		return;
> +
> +	xe_gt_sriov_notice(gt, "VF%u unprocessed migration data left in the ring!\n", vfid);
> +
> +	while ((data = ptr_ring_consume(&migration->ring)))
> +		xe_sriov_packet_free(data);
> +}
> +
>  /**
>   * xe_gt_sriov_pf_migration_save_produce() - Add VF save data packet to migration ring.
>   * @gt: the &xe_gt
> @@ -544,8 +564,10 @@ xe_gt_sriov_pf_migration_save_consume(struct xe_gt *gt, unsigned int vfid)
>  	data = ptr_ring_consume(&migration->ring);
>  	if (data) {
>  		ret = xe_gt_sriov_pf_control_process_save_data(gt, vfid);
> -		if (ret)
> +		if (ret) {
> +			xe_sriov_packet_free(data);
>  			return ERR_PTR(ret);
> +		}
>  
>  		return data;
>  	}
> @@ -559,11 +581,18 @@ xe_gt_sriov_pf_migration_save_consume(struct xe_gt *gt, unsigned int vfid)
>  	return ERR_PTR(-EAGAIN);
>  }
>  
> +static void pf_mig_packet_destroy(void *ptr)

it's not a "pf" call, but 'destroy' callback, so maybe:

	destroy_pf_packet()

> +{
> +	struct xe_sriov_packet *data = ptr;
> +
> +	xe_sriov_packet_free(data);
> +}
> +
>  static void action_ring_cleanup(void *arg)
>  {
>  	struct ptr_ring *r = arg;
>  
> -	ptr_ring_cleanup(r, NULL);
> +	ptr_ring_cleanup(r, pf_mig_packet_destroy);
>  }
>  
>  /**
> diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h
> index b29d34a835b71..b4c7b5dc9e914 100644
> --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h
> +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.h
> @@ -17,6 +17,7 @@ int xe_gt_sriov_pf_migration_restore_guc_state(struct xe_gt *gt, unsigned int vf
>  
>  bool xe_gt_sriov_pf_migration_ring_empty(struct xe_gt *gt, unsigned int vfid);
>  bool xe_gt_sriov_pf_migration_ring_full(struct xe_gt *gt, unsigned int vfid);
> +void xe_gt_sriov_pf_migration_ring_free(struct xe_gt *gt, unsigned int vfid);
>  
>  int xe_gt_sriov_pf_migration_save_produce(struct xe_gt *gt, unsigned int vfid,
>  					  struct xe_sriov_packet *data);
> diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.c b/drivers/gpu/drm/xe/xe_sriov_packet.c
> new file mode 100644
> index 0000000000000..2cdcfb2073d00
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_sriov_packet.c
> @@ -0,0 +1,129 @@
> +// SPDX-License-Identifier: MIT
> +/*
> + * Copyright © 2025 Intel Corporation
> + */
> +
> +#include "xe_bo.h"
> +#include "xe_device.h"
> +#include "xe_sriov_packet.h"
> +
> +static bool pkt_needs_bo(struct xe_sriov_packet *data)
> +{
> +	return data->type == XE_SRIOV_PACKET_TYPE_VRAM;
> +}
> +
> +/**
> + * xe_sriov_packet_alloc() - Allocate migration data packet
> + * @xe: the &xe_device
> + *
> + * Only allocates the "outer" structure, without initializing the migration
> + * data backing storage.
> + *
> + * Return: Pointer to &xe_sriov_packet on success,
> + *         NULL in case of error.
> + */
> +struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe)
> +{
> +	struct xe_sriov_packet *data;
> +
> +	data = kzalloc(sizeof(*data), GFP_KERNEL);
> +	if (!data)
> +		return NULL;
> +
> +	data->xe = xe;
> +	data->hdr_remaining = sizeof(data->hdr);
> +
> +	return data;
> +}
> +
> +/**
> + * xe_sriov_packet_free() - Free migration data packet.
> + * @data: the &xe_sriov_packet
> + */
> +void xe_sriov_packet_free(struct xe_sriov_packet *data)
> +{
> +	if (IS_ERR_OR_NULL(data))
> +		return;
> +
> +	if (pkt_needs_bo(data))
> +		xe_bo_unpin_map_no_vm(data->bo);
> +	else
> +		kvfree(data->buff);
> +
> +	kfree(data);
> +}
> +
> +static int mig_pkt_init(struct xe_sriov_packet *data)

nit: no need for "mig_" prefix, see above pkt_needs_bo()

> +{
> +	struct xe_gt *gt = xe_device_get_gt(data->xe, data->gt);

xe_device_get_gt() may return NULL and this will make static code analyzers unhappy

either we should check here gt again, or maybe store the valid (struct xe_gt *) in the data,
as we already have (struct xe_device *data.xe)

> +
> +	if (data->size == 0)
> +		return 0;
> +
> +	if (pkt_needs_bo(data)) {
> +		struct xe_bo *bo;
> +
> +		bo = xe_bo_create_pin_map_novm(data->xe, gt->tile, PAGE_ALIGN(data->size),
> +					       ttm_bo_type_kernel,
> +					       XE_BO_FLAG_SYSTEM | XE_BO_FLAG_PINNED, false);
> +		if (IS_ERR(bo))
> +			return PTR_ERR(bo);
> +
> +		data->bo = bo;
> +		data->vaddr = bo->vmap.vaddr;
> +	} else {
> +		void *buff = kvzalloc(data->size, GFP_KERNEL);
> +
> +		if (!buff)
> +			return -ENOMEM;
> +
> +		data->buff = buff;
> +		data->vaddr = buff;
> +	}
> +
> +	return 0;
> +}
> +
> +#define XE_SRIOV_PACKET_SUPPORTED_VERSION 1

nit: add new line here

> +/**
> + * xe_sriov_packet_init() - Initialize migration packet header and backing storage.
> + * @data: the &xe_sriov_packet
> + * @tile_id: tile identifier
> + * @gt_id: GT identifier
> + * @type: &xe_sriov_packet_type
> + * @offset: offset of data packet payload (within wider resource)
> + * @size: size of data packet payload
> + *
> + * Return: 0 on success or a negative error code on failure.
> + */
> +int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id,
> +			 enum xe_sriov_packet_type type, loff_t offset, size_t size)
> +{
> +	data->version = XE_SRIOV_PACKET_SUPPORTED_VERSION;
> +	data->type = type;
> +	data->tile = tile_id;
> +	data->gt = gt_id;

shouldn't we validate the tile_id / gt_id here?

we do have valid data->xe here, so we can try:

	data->gt = xe_device_get_gt(data->xe, gt_id);
	if (data->gt)
		return -ENODEV;
	data->hdr.gt_id = gt_id;
	...


> +	data->offset = offset;
> +	data->size = size;
> +	data->remaining = size;
> +
> +	return mig_pkt_init(data);
> +}
> +
> +/**
> + * xe_sriov_packet_init_from_hdr() - Initialize migration packet backing storage based on header.
> + * @data: the &xe_sriov_packet
> + *
> + * Header data is expected to be filled prior to calling this function.

maybe add some asserts to enforce that and check data->hdr.size and other fields?

> + *
> + * Return: 0 on success or a negative error code on failure.
> + */
> +int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data)
> +{
> +	if (data->version != XE_SRIOV_PACKET_SUPPORTED_VERSION)
> +		return -EINVAL;
> +
> +	data->remaining = data->size;
> +
> +	return mig_pkt_init(data);
> +}
> diff --git a/drivers/gpu/drm/xe/xe_sriov_packet.h b/drivers/gpu/drm/xe/xe_sriov_packet.h
> new file mode 100644
> index 0000000000000..2c5a49897d763
> --- /dev/null
> +++ b/drivers/gpu/drm/xe/xe_sriov_packet.h
> @@ -0,0 +1,30 @@
> +/* SPDX-License-Identifier: MIT */
> +/*
> + * Copyright © 2025 Intel Corporation
> + */
> +
> +#ifndef _XE_SRIOV_PACKET_H_
> +#define _XE_SRIOV_PACKET_H_
> +
> +#include <linux/types.h>
> +
> +struct xe_device;

also

   struct xe_sriov_packet;

> +
> +enum xe_sriov_packet_type {
> +	/* Skipping 0 to catch uninitialized data */
> +	XE_SRIOV_PACKET_TYPE_DESCRIPTOR = 1,
> +	XE_SRIOV_PACKET_TYPE_TRAILER,
> +	XE_SRIOV_PACKET_TYPE_GGTT,
> +	XE_SRIOV_PACKET_TYPE_MMIO,
> +	XE_SRIOV_PACKET_TYPE_GUC,
> +	XE_SRIOV_PACKET_TYPE_VRAM,
> +};

shouldn't this enum be in xe_sriov_packet_types.h too ?

> +
> +struct xe_sriov_packet *xe_sriov_packet_alloc(struct xe_device *xe);
> +void xe_sriov_packet_free(struct xe_sriov_packet *data);
> +
> +int xe_sriov_packet_init(struct xe_sriov_packet *data, u8 tile_id, u8 gt_id,
> +			 enum xe_sriov_packet_type, loff_t offset, size_t size);
> +int xe_sriov_packet_init_from_hdr(struct xe_sriov_packet *data);
> +
> +#endif