[PATCH] mm/migrate: look up consecutive pages together in do_pages_stat_array()

[PATCH] mm/migrate: look up consecutive pages together in do_pages_stat_array()

[Qiliang Yuan]

move_pages() with a NULL node list reports the node of each page. RDMA
and KV-cache transfer engines use it to find where large registered
buffers live, querying every 4K page of buffers that span hundreds of
gigabytes.

do_pages_stat_array() looks up the VMA and walks the page tables from
the top for every address, taking and dropping the PTE lock each time.
That costs about 105 ns per page, so a 16 GiB buffer takes 440 ms.

Callers almost always pass consecutive addresses. Reuse the VMA while
the address stays inside it. After folio_walk_start() locks a PTE table
or a PMD-mapped folio, answer the following addresses under the same
PMD from that table or folio before releasing the lock. Other
addresses still take the existing path.

On 7.3-rc5 in a 16-vCPU VM, querying every page of a populated buffer:

                       before       after
  4K pages, per page   105 ns       22.7 ns
  THP, per page        92 ns        13.1 ns
  16 GiB, 4K pages     439 ms       95 ms

Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
---
 mm/migrate.c | 83 ++++++++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 61 insertions(+), 22 deletions(-)

diff --git a/mm/migrate.c b/mm/migrate.c
index 15b45832bcfa7..5f470dc7dc746 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -2451,44 +2451,83 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
 	return err;
 }
 
+static int folio_stat(struct folio *folio)
+{
+	if (is_zero_folio(folio) || is_huge_zero_folio(folio))
+		return -EFAULT;
+	if (folio_is_zone_device(folio))
+		return -ENOENT;
+	return folio_nid(folio);
+}
+
+/* Look up a PTE in a locked page table the way folio_walk_start() does. */
+static int pte_stat(struct vm_area_struct *vma, unsigned long addr,
+		    pte_t *ptep)
+{
+	pte_t pte = ptep_get(ptep);
+	struct page *page;
+
+	if (!pte_present(pte))
+		return -ENOENT;
+	page = vm_normal_page(vma, addr, pte);
+	if (page)
+		return folio_stat(page_folio(page));
+	if (is_zero_pfn(pte_pfn(pte)))
+		return -EFAULT;
+	return -ENOENT;
+}
+
 /*
  * Determine the nodes of an array of pages and store it in an array of status.
  */
 static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
 				const void __user **pages, int *status)
 {
-	unsigned long i;
+	struct vm_area_struct *vma = NULL;
+	unsigned long i = 0;
 
 	mmap_read_lock(mm);
 
-	for (i = 0; i < nr_pages; i++) {
-		unsigned long addr = (unsigned long)(*pages);
-		struct vm_area_struct *vma;
+	while (i < nr_pages) {
+		unsigned long addr = (unsigned long)pages[i];
+		unsigned long next, end;
 		struct folio_walk fw;
 		struct folio *folio;
-		int err = -EFAULT;
+		pte_t *ptep;
 
-		vma = vma_lookup(mm, addr);
-		if (!vma)
-			goto set_status;
+		if (!vma || addr < vma->vm_start || addr >= vma->vm_end)
+			vma = vma_lookup(mm, addr);
+		if (!vma) {
+			status[i++] = -EFAULT;
+			continue;
+		}
 
 		folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
-		if (folio) {
-			if (is_zero_folio(folio) || is_huge_zero_folio(folio))
-				err = -EFAULT;
-			else if (folio_is_zone_device(folio))
-				err = -ENOENT;
-			else
-				err = folio_nid(folio);
-			folio_walk_end(&fw, vma);
-		} else {
-			err = -ENOENT;
+		if (!folio) {
+			status[i++] = -ENOENT;
+			continue;
 		}
-set_status:
-		*status = err;
+		status[i++] = folio_stat(folio);
 
-		pages++;
-		status++;
+		/*
+		 * Callers usually pass consecutive pages. Answer those that
+		 * fall under the entry or page table we already hold locked
+		 * instead of walking the page tables again for each of them.
+		 */
+		end = pmd_addr_end(addr, vma->vm_end);
+		ptep = fw.ptep;
+		for (next = addr + PAGE_SIZE; i < nr_pages && next < end;
+		     next += PAGE_SIZE, i++) {
+			if ((unsigned long)pages[i] != next)
+				break;
+			if (fw.level == FW_LEVEL_PMD)
+				status[i] = status[i - 1];
+			else if (fw.level == FW_LEVEL_PTE)
+				status[i] = pte_stat(vma, next, ++ptep);
+			else
+				break;
+		}
+		folio_walk_end(&fw, vma);
 	}
 
 	mmap_read_unlock(mm);

---
base-commit: 551c722f40809618230001baccf219193e22fc5a
change-id: 20261001-bug-mm-move-pages-stat-batch-f62a29ea5867

Best regards,
-- 
Qiliang Yuan <odys.yuan@gmail.com>

Re: [PATCH] mm/migrate: look up consecutive pages together in do_pages_stat_array()

[Zi Yan]

On 1 Oct 2026, at 11:10, Qiliang Yuan wrote:

> move_pages() with a NULL node list reports the node of each page. RDMA
> and KV-cache transfer engines use it to find where large registered
> buffers live, querying every 4K page of buffers that span hundreds of
> gigabytes.
>
> do_pages_stat_array() looks up the VMA and walks the page tables from
> the top for every address, taking and dropping the PTE lock each time.
> That costs about 105 ns per page, so a 16 GiB buffer takes 440 ms.
>
> Callers almost always pass consecutive addresses. Reuse the VMA while
> the address stays inside it. After folio_walk_start() locks a PTE table
> or a PMD-mapped folio, answer the following addresses under the same
> PMD from that table or folio before releasing the lock. Other
> addresses still take the existing path.

Instead of adding customized batched walk code, it might be better to use
walk_page_range() + pmd_entry callback. See mincore_pte_range() for
reference.

BTW, DO_PAGES_STAT_CHUNK_NR is limited at 16, so the batching benefit
comes from 16 pages instead of 512 pages in one PMD? I wonder if that
limit could be raised and how much more benefit we will see.

>
> On 7.3-rc5 in a 16-vCPU VM, querying every page of a populated buffer:
>
>                        before       after
>   4K pages, per page   105 ns       22.7 ns
>   THP, per page        92 ns        13.1 ns
>   16 GiB, 4K pages     439 ms       95 ms
>
> Signed-off-by: Qiliang Yuan <odys.yuan@gmail.com>
> ---
>  mm/migrate.c | 83 ++++++++++++++++++++++++++++++++++++++++++++----------------
>  1 file changed, 61 insertions(+), 22 deletions(-)
>
> diff --git a/mm/migrate.c b/mm/migrate.c
> index 15b45832bcfa7..5f470dc7dc746 100644
> --- a/mm/migrate.c
> +++ b/mm/migrate.c
> @@ -2451,44 +2451,83 @@ static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
>  	return err;
>  }
>
> +static int folio_stat(struct folio *folio)
> +{
> +	if (is_zero_folio(folio) || is_huge_zero_folio(folio))
> +		return -EFAULT;
> +	if (folio_is_zone_device(folio))
> +		return -ENOENT;
> +	return folio_nid(folio);
> +}
> +
> +/* Look up a PTE in a locked page table the way folio_walk_start() does. */
> +static int pte_stat(struct vm_area_struct *vma, unsigned long addr,
> +		    pte_t *ptep)
> +{
> +	pte_t pte = ptep_get(ptep);
> +	struct page *page;
> +
> +	if (!pte_present(pte))
> +		return -ENOENT;
> +	page = vm_normal_page(vma, addr, pte);
> +	if (page)
> +		return folio_stat(page_folio(page));
> +	if (is_zero_pfn(pte_pfn(pte)))
> +		return -EFAULT;
> +	return -ENOENT;
> +}
> +
>  /*
>   * Determine the nodes of an array of pages and store it in an array of status.
>   */
>  static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
>  				const void __user **pages, int *status)
>  {
> -	unsigned long i;
> +	struct vm_area_struct *vma = NULL;
> +	unsigned long i = 0;
>
>  	mmap_read_lock(mm);
>
> -	for (i = 0; i < nr_pages; i++) {
> -		unsigned long addr = (unsigned long)(*pages);
> -		struct vm_area_struct *vma;
> +	while (i < nr_pages) {
> +		unsigned long addr = (unsigned long)pages[i];
> +		unsigned long next, end;
>  		struct folio_walk fw;
>  		struct folio *folio;
> -		int err = -EFAULT;
> +		pte_t *ptep;
>
> -		vma = vma_lookup(mm, addr);
> -		if (!vma)
> -			goto set_status;
> +		if (!vma || addr < vma->vm_start || addr >= vma->vm_end)
> +			vma = vma_lookup(mm, addr);
> +		if (!vma) {
> +			status[i++] = -EFAULT;
> +			continue;
> +		}
>
>  		folio = folio_walk_start(&fw, vma, addr, FW_ZEROPAGE);
> -		if (folio) {
> -			if (is_zero_folio(folio) || is_huge_zero_folio(folio))
> -				err = -EFAULT;
> -			else if (folio_is_zone_device(folio))
> -				err = -ENOENT;
> -			else
> -				err = folio_nid(folio);
> -			folio_walk_end(&fw, vma);
> -		} else {
> -			err = -ENOENT;
> +		if (!folio) {
> +			status[i++] = -ENOENT;
> +			continue;
>  		}
> -set_status:
> -		*status = err;
> +		status[i++] = folio_stat(folio);
>
> -		pages++;
> -		status++;
> +		/*
> +		 * Callers usually pass consecutive pages. Answer those that
> +		 * fall under the entry or page table we already hold locked
> +		 * instead of walking the page tables again for each of them.
> +		 */
> +		end = pmd_addr_end(addr, vma->vm_end);
> +		ptep = fw.ptep;
> +		for (next = addr + PAGE_SIZE; i < nr_pages && next < end;
> +		     next += PAGE_SIZE, i++) {
> +			if ((unsigned long)pages[i] != next)
> +				break;
> +			if (fw.level == FW_LEVEL_PMD)
> +				status[i] = status[i - 1];
> +			else if (fw.level == FW_LEVEL_PTE)
> +				status[i] = pte_stat(vma, next, ++ptep);
> +			else
> +				break;
> +		}
> +		folio_walk_end(&fw, vma);
>  	}
>
>  	mmap_read_unlock(mm);
>
> ---
> base-commit: 551c722f40809618230001baccf219193e22fc5a
> change-id: 20261001-bug-mm-move-pages-stat-batch-f62a29ea5867
>
> Best regards,
> -- 
> Qiliang Yuan <odys.yuan@gmail.com>


Best Regards,
Yan, Zi