Re: [PATCH v3 19/33] drm/xe: Skip self-copies for borrowed pages on defrag moves

[Niranjana Vishwanathapura <niranjana.vishwanathapura@intel.com>]

On Fri, Jul 10, 2026 at 07:56:05PM -0700, Matthew Brost wrote:
>A defrag move may harvest already beneficial-order chunks from the old
>tt, so the freshly reallocated backing borrows (references) some of the
>same physical pages. For those pages the source and destination GPU
>mappings alias the same memory, making the data blit a pointless
>self-copy.
>
>Pass the source (old) tt into __xe_migrate_copy() instead of just its sg
>table, so the data pass can compare the src and dst tt page arrays at the
>matching offset and only blit runs of pages that actually differ. A new
>emit_copy_chunks() helper walks each L0 chunk, skips borrowed-page runs,
>and coalesces contiguous to-copy pages into a single emit_copy(). It runs
>twice per chunk: once with a NULL batch buffer to count the blits for
>batch sizing, then again to emit them.
>
>When an entire L0 chunk turns out to be borrowed, skip it outright: no
>batch buffer is allocated and no GPU commands are issued, the cursors are
>just advanced past the chunk. If every chunk of the move is borrowed the
>copy is a complete no-op, so hand back a signaled stub fence.
>
>The comparison is a cheap per-page pointer test (src page != dst page)
>that coalesces contiguous to-copy pages into a single blit. PTEs are
>still emitted for partially borrowed chunks, since that setup is cheap and
>only the data copies are worth skipping. Only the defrag data pass is
>affected; the separate CCS pass still migrates all aux state.
>
>Since the Xe tt sg lives in the xe_ttm_tt subclass rather than the base
>ttm_tt, add an xe_tt_sg() helper to fetch it from a struct ttm_tt.
>
>Cc: Carlos Santa <carlos.santa@intel.com>
>Cc: Ryan Neph <ryanneph@google.com>
>Cc: Christian Koenig <christian.koenig@amd.com>
>Cc: Huang Rui <ray.huang@amd.com>
>Cc: Matthew Auld <matthew.auld@intel.com>
>Cc: Maarten Lankhorst <maarten.lankhorst@linux.intel.com>
>Cc: Maxime Ripard <mripard@kernel.org>
>Cc: Thomas Zimmermann <tzimmermann@suse.de>
>Cc: David Airlie <airlied@gmail.com>
>Cc: Simona Vetter <simona@ffwll.ch>
>Cc: dri-devel@lists.freedesktop.org
>Cc: linux-kernel@vger.kernel.org
>Cc: Thomas Hellström <thomas.hellstrom@linux.intel.com>
>Assisted-by: GitHub_Copilot:claude-opus-4.8
>Signed-off-by: Matthew Brost <matthew.brost@intel.com>
>
>----
>
>v3:
> - Fix start_page page-shift and blit pitch for src-natural to
>   dst-packed copies (Sashiko)
>---
> drivers/gpu/drm/xe/xe_bo.c      |  22 +++++-
> drivers/gpu/drm/xe/xe_bo.h      |   2 +
> drivers/gpu/drm/xe/xe_migrate.c | 136 ++++++++++++++++++++++++++++----
> drivers/gpu/drm/xe/xe_migrate.h |   3 +-
> 4 files changed, 142 insertions(+), 21 deletions(-)
>
>diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c
>index bdcc9bed5d36..7f6ee9b8d12e 100644
>--- a/drivers/gpu/drm/xe/xe_bo.c
>+++ b/drivers/gpu/drm/xe/xe_bo.c
>@@ -473,6 +473,22 @@ struct sg_table *xe_bo_sg(struct xe_bo *bo)
> 	return xe_tt->sg;
> }
>
>+/**
>+ * xe_tt_sg() - Return the DMA scatter-gather table backing a ttm_tt.
>+ * @tt: The ttm_tt to query.
>+ *
>+ * Like xe_bo_sg(), but takes the ttm_tt directly. Used by the defrag copy
>+ * path which needs the old (source) tt's sg table without a backing xe_bo.
>+ *
>+ * Return: The sg table, or NULL if the tt is not currently mapped.
>+ */
>+struct sg_table *xe_tt_sg(struct ttm_tt *tt)
>+{
>+	struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm);
>+
>+	return xe_tt->sg;
>+}

Assisted-by: LLM

xe_tt->sg can be potentially NULL (though probably not in current usecase).
Probably an explicit assert better?

>+
> /*
>  * Account ttm pages against the device shrinker's shrinkable and
>  * purgeable counts.
>@@ -1290,11 +1306,9 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict,
>
> 		fence = xe_migrate_clear(migrate, bo, new_mem, flags);
> 	} else if (ttm_bo->defrag_old_tt) {
>-		struct xe_ttm_tt *old_xe_tt =
>-			container_of(ttm_bo->defrag_old_tt, struct xe_ttm_tt, ttm);
>-
> 		fence = xe_migrate_copy_defrag(migrate, bo, old_mem, new_mem,
>-					       old_xe_tt->sg, handle_system_ccs);
>+					       ttm_bo->defrag_old_tt,
>+					       handle_system_ccs);
> 	} else {
> 		fence = xe_migrate_copy(migrate, bo, bo, old_mem, new_mem,
> 					handle_system_ccs);
>diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h
>index 4186cb51af87..32211016a722 100644
>--- a/drivers/gpu/drm/xe/xe_bo.h
>+++ b/drivers/gpu/drm/xe/xe_bo.h
>@@ -110,6 +110,7 @@ enum xe_madv_purgeable_state {
> };
>
> struct sg_table;
>+struct ttm_tt;
>
> struct xe_bo *xe_bo_alloc(void);
> void xe_bo_free(struct xe_bo *bo);
>@@ -547,6 +548,7 @@ void xe_bo_dev_init(struct xe_bo_dev *bo_device);
> void xe_bo_dev_fini(struct xe_bo_dev *bo_device);
>
> struct sg_table *xe_bo_sg(struct xe_bo *bo);
>+struct sg_table *xe_tt_sg(struct ttm_tt *tt);
>
> /*
>  * xe_sg_segment_size() - Provides upper limit for sg segment size.
>diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c
>index 305c6e1942fb..9990a7144a9f 100644
>--- a/drivers/gpu/drm/xe/xe_migrate.c
>+++ b/drivers/gpu/drm/xe/xe_migrate.c
>@@ -811,6 +811,63 @@ static void emit_copy(struct xe_gt *gt, struct xe_bb *bb,
> 		emit_xy_fast_copy(gt, bb, src_ofs, dst_ofs, size, pitch);
> }
>
>+/*
>+ * emit_copy_chunks() - Emit the data blits for one defrag L0 chunk, skipping
>+ * borrowed pages.
>+ * @gt: The GT.
>+ * @bb: The batch buffer to emit into, or NULL to only count the blits needed.
>+ * @src_L0_ofs: GPU offset of the source pages mapped for this L0 chunk.
>+ * @dst_L0_ofs: GPU offset of the destination pages mapped for this L0 chunk.
>+ * @src_L0: Byte size of the L0 chunk (page aligned).
>+ * @src_tt: Source (old) tt providing the page pointers.
>+ * @dst_tt: Destination (new) tt providing the page pointers.
>+ * @start_page: Page offset of this L0 chunk within the tt page arrays.
>+ *
>+ * A defragmentation move may harvest already beneficial-order chunks from the
>+ * old tt, so the new tt references (borrows) some of the same physical pages.
>+ * For those pages the source and destination GPU mappings alias the same
>+ * memory and the data blit is a pointless self-copy. Walk the L0 chunk and
>+ * only blit runs of pages that actually differ (src page != dst page),
>+ * coalescing contiguous to-copy pages into a single blit.
>+ *
>+ * Return: The number of blits emitted (or that would be emitted when @bb is
>+ * NULL).
>+ */
>+static u32 emit_copy_chunks(struct xe_gt *gt, struct xe_bb *bb,
>+			    u64 src_L0_ofs, u64 dst_L0_ofs, u32 src_L0,
>+			    struct ttm_tt *src_tt, struct ttm_tt *dst_tt,
>+			    pgoff_t start_page)
>+{

emit_copy() doesn't have L0 in parameter names. Probably it is worth keeping
it consistent here also?

>+	u32 npages = src_L0 >> PAGE_SHIFT;
>+	u32 i = 0, nblits = 0;
>+
>+	while (i < npages) {
>+		u32 run = 0;
>+
>+		/* Skip a run of borrowed pages (src and dst alias). */
>+		if (src_tt->pages[start_page + i] ==
>+		    dst_tt->pages[start_page + i]) {
>+			i++;
>+			continue;
>+		}
>+
>+		/* Coalesce a run of pages that actually need copying. */
>+		while (i + run < npages &&
>+		       src_tt->pages[start_page + i + run] !=
>+		       dst_tt->pages[start_page + i + run])
>+			run++;
>+
>+		if (bb)
>+			emit_copy(gt, bb, src_L0_ofs + (u64)i * PAGE_SIZE,
>+				  dst_L0_ofs + (u64)i * PAGE_SIZE,
>+				  run * PAGE_SIZE, XE_PAGE_SIZE);
>+		nblits++;
>+		i += run;
>+	}
>+
>+	return nblits;
>+}

Would it be good or possible to have similar counterpart for emit_copy_ccs()?

>+
> static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm)
> {
> 	return usm ? m->usm_batch_base_ofs : m->batch_base_ofs;
>@@ -865,13 +922,15 @@ static u32 xe_migrate_ccs_copy(struct xe_migrate *m,
>  * @is_vram_resolve: Decompression (VRAM -> system) resolve; always uses the
>  * compression PAT index for the data copy.
>  * @defrag_ccs: Defrag move pass that migrates only the CCS aux state between
>- * two distinct page sets (old @src_sg pages -> new pages). The data blit is
>+ * two distinct page sets (old @src_tt pages -> new pages). The data blit is
>  * skipped (the data was copied by the @defrag_copy pass) and both the source
>  * and destination data pages are mapped with the compression PAT so the aux
>  * state is copied indirect -> indirect.
>  * @defrag_copy: Defrag move pass that copies the data pages between two
>- * distinct page sets (old @src_sg pages -> new pages) without touching the CCS
>- * aux state. The data pages are mapped without the compression PAT so the
>+ * distinct page sets (old @src_tt pages -> new pages) without touching the CCS
>+ * aux state. Pages the new tt borrowed from @src_tt (harvested already
>+ * beneficial-order chunks) alias the same memory on both sides and are skipped.
>+ * The data pages are mapped without the compression PAT so the
>  * (possibly compressed) bytes are copied verbatim rather than being
>  * decompressed and recompressed, and no CCS aux copy is appended; the aux
>  * state is migrated separately by the @defrag_ccs pass. The two passes need
>@@ -889,7 +948,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 					   struct xe_bo *dst_bo,
> 					   struct ttm_resource *src,
> 					   struct ttm_resource *dst,
>-					   struct sg_table *src_sg,
>+					   struct ttm_tt *src_tt,
> 					   struct xe_migrate_copy_flags flags)
> {
> 	struct xe_gt *gt = m->tile->primary_gt;
>@@ -914,6 +973,14 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		!flags.defrag_copy;
> 	bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram) &&
> 		!flags.defrag_ccs && !flags.defrag_copy;
>+	/*
>+	 * On the defrag data pass the new (destination) tt may have borrowed
>+	 * already beneficial-order pages from the old (source) tt, so skip the
>+	 * self-copy for any page they share. Compare the tt page arrays at the
>+	 * matching offset.
>+	 */
>+	struct ttm_tt *dst_tt = dst_bo->ttm.ttm;
>+	bool skip_borrowed = flags.defrag_copy && src_tt && dst_tt;

I think for our case src_tt and and dst_tt check is redundant here as they will
always be set for defrag_copy. May be add an assert instead of checking?

Niranjana

>
> 	/*
> 	 * For decompression operation, always use the compression PAT index.
>@@ -926,7 +993,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
>
> 	/*
> 	 * The defrag CCS pass migrates the aux state from the old pages
>-	 * (@src_sg) to the new pages with both sides accessed indirectly, so
>+	 * (@src_tt) to the new pages with both sides accessed indirectly, so
> 	 * keep both src_is_pltt and dst_is_pltt set (no direct staging side).
> 	 */
>
>@@ -938,7 +1005,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		return ERR_PTR(-EINVAL);
>
> 	if (!src_is_vram)
>-		xe_res_first_sg(src_sg ? src_sg : xe_bo_sg(src_bo), 0, size, &src_it);
>+		xe_res_first_sg(src_tt ? xe_tt_sg(src_tt) : xe_bo_sg(src_bo), 0, size, &src_it);
> 	else
> 		xe_res_first(src, 0, size, &src_it);
> 	if (!dst_is_vram)
>@@ -947,7 +1014,7 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		xe_res_first(dst, 0, size, &dst_it);
>
> 	if (copy_system_ccs)
>-		xe_res_first_sg(src_sg ? src_sg : xe_bo_sg(src_bo),
>+		xe_res_first_sg(src_tt ? xe_tt_sg(src_tt) : xe_bo_sg(src_bo),
> 				xe_bo_ccs_pages_start(src_bo),
> 				PAGE_ALIGN(xe_device_ccs_bytes(xe, size)),
> 				&ccs_it);
>@@ -961,6 +1028,8 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		u64 ccs_ofs, ccs_size;
> 		u32 ccs_pt;
> 		u32 pte_flags;
>+		pgoff_t start_page = (xe_bo_size(src_bo) - size) >> PAGE_SHIFT;
>+		u32 ndata_blits = 1;
>
> 		bool usm = xe->info.has_usm;
> 		u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE;
>@@ -998,9 +1067,30 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 			xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE));
> 		}
>
>-		/* Add copy commands size here */
>+		/*
>+		 * Add copy commands size here. On the defrag data pass the blit
>+		 * is split into one command per run of non-borrowed pages, so
>+		 * size for the number of runs actually needed.
>+		 */
>+		if (skip_borrowed)
>+			ndata_blits = emit_copy_chunks(gt, NULL, 0, 0, src_L0,
>+						       src_tt, dst_tt, start_page);
>+
>+		/*
>+		 * If every page in this chunk was borrowed there is nothing to
>+		 * copy: don't build a batch or issue any GPU commands, just
>+		 * advance the cursors past the chunk. The non-vram tt cursors
>+		 * would otherwise be advanced by emit_pte().
>+		 */
>+		if (skip_borrowed && !ndata_blits) {
>+			xe_res_next(&src_it, src_L0);
>+			xe_res_next(&dst_it, src_L0);
>+			size -= src_L0;
>+			continue;
>+		}
>+
> 		batch_size += ((flags.copy_only_ccs || flags.defrag_ccs) ? 0 :
>-			       emit_copy_cmd_len(xe)) +
>+			       ndata_blits * emit_copy_cmd_len(xe)) +
> 			((needs_ccs_emit ? EMIT_COPY_CCS_DW : 0));
>
> 		bb = xe_bb_new(gt, batch_size, usm);
>@@ -1029,7 +1119,10 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		bb->cs[bb->len++] = MI_BATCH_BUFFER_END;
> 		update_idx = bb->len;
>
>-		if (!flags.copy_only_ccs && !flags.defrag_ccs)
>+		if (skip_borrowed)
>+			emit_copy_chunks(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0,
>+					 src_tt, dst_tt, start_page);
>+		else if (!flags.copy_only_ccs && !flags.defrag_ccs)
> 			emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE);
>
> 		if (needs_ccs_emit)
>@@ -1088,6 +1181,14 @@ static struct dma_fence *__xe_migrate_copy(struct xe_migrate *m,
> 		return ERR_PTR(err);
> 	}
>
>+	/*
>+	 * Every chunk of a defrag data pass may have been borrowed (nothing
>+	 * copied), leaving no job and a NULL fence. Hand back a signaled stub
>+	 * so callers always get a valid fence to wait on.
>+	 */
>+	if (!fence)
>+		return dma_fence_get_stub();
>+
> 	return fence;
> }
>
>@@ -1129,18 +1230,21 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m,
>  * @bo: The buffer object being defragmented. Both @src and @dst belong to @bo.
>  * @src: The source TTM resource (the old backing).
>  * @dst: The dst TTM resource (the freshly reallocated backing).
>- * @src_sg: Scatter-gather table of the source (old) backing pages.
>+ * @src_tt: The source (old) backing tt, providing both the scatter-gather
>+ * table and page pointers of the old pages.
>  * @need_ccs: Whether the BO has CCS aux state that must be migrated too.
>  *
>  * Used by the defrag path where a single BO has its backing reallocated at a
>  * more beneficial page order. The contents are relocated from the old pages
>- * (@src_sg) to the new pages (the BO's current backing) entirely on the GPU,
>+ * (@src_tt) to the new pages (the BO's current backing) entirely on the GPU,
>  * in up to two passes:
>  *
>  *  - Data pass (always): copy the data pages verbatim. The pages are mapped
>  *    without the compression PAT, so the (possibly compressed) bytes are copied
>  *    as-is rather than being decompressed and recompressed, and no CCS aux copy
>- *    is appended.
>+ *    is appended. Pages the new backing borrowed from @src_tt (harvested
>+ *    already beneficial-order chunks) alias the same memory on both sides and
>+ *    are skipped.
>  *
>  *  - CCS pass (@need_ccs): copy the CCS aux (compression) state. Both the old
>  *    and new data pages are mapped with the compression PAT so the copy engine
>@@ -1159,11 +1263,11 @@ struct dma_fence *xe_migrate_copy_defrag(struct xe_migrate *m,
> 					 struct xe_bo *bo,
> 					 struct ttm_resource *src,
> 					 struct ttm_resource *dst,
>-					 struct sg_table *src_sg,
>+					 struct ttm_tt *src_tt,
> 					 bool need_ccs)
> {
> 	struct dma_fence *fence2, *fence =
>-		__xe_migrate_copy(m, bo, bo, src, dst, src_sg,
>+		__xe_migrate_copy(m, bo, bo, src, dst, src_tt,
> 				  (struct xe_migrate_copy_flags) {
> 					 .defrag_copy = true,
> 					 });
>@@ -1171,7 +1275,7 @@ struct dma_fence *xe_migrate_copy_defrag(struct xe_migrate *m,
> 	if (IS_ERR(fence) || !need_ccs)
> 		return fence;
>
>-	fence2 = __xe_migrate_copy(m, bo, bo, src, dst, src_sg,
>+	fence2 = __xe_migrate_copy(m, bo, bo, src, dst, src_tt,
> 				   (struct xe_migrate_copy_flags) {
> 				   .defrag_ccs = true,
> 				   });
>diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h
>index e4f996942a6d..4094ea1c19c1 100644
>--- a/drivers/gpu/drm/xe/xe_migrate.h
>+++ b/drivers/gpu/drm/xe/xe_migrate.h
>@@ -13,6 +13,7 @@ struct drm_pagemap_addr;
> struct iosys_map;
> struct sg_table;
> struct ttm_resource;
>+struct ttm_tt;
>
> struct xe_bo;
> struct xe_gt;
>@@ -137,7 +138,7 @@ struct dma_fence *xe_migrate_copy_defrag(struct xe_migrate *m,
> 					 struct xe_bo *bo,
> 					 struct ttm_resource *src,
> 					 struct ttm_resource *dst,
>-					 struct sg_table *src_sg,
>+					 struct ttm_tt *src_tt,
> 					 bool need_ccs);
>
> struct dma_fence *xe_migrate_resolve(struct xe_migrate *m,
>-- 
>2.34.1
>