mirror of https://lore.kernel.org/lkml/
 help / color / mirror / Atom feed
From: Andrea Righi <arighi@nvidia.com>
To: Alexei Starovoitov <ast@kernel.org>,
	Daniel Borkmann <daniel@iogearbox.net>,
	Andrii Nakryiko <andrii@kernel.org>,
	Eduard Zingerman <eddyz87@gmail.com>,
	Kumar Kartikeya Dwivedi <memxor@gmail.com>
Cc: Martin KaFai Lau <martin.lau@linux.dev>,
	Song Liu <song@kernel.org>,
	Yonghong Song <yonghong.song@linux.dev>,
	Jiri Olsa <jolsa@kernel.org>,
	Emil Tsalapatis <emil@etsalapatis.com>,
	Ihor Solodrai <ihor.solodrai@linux.dev>,
	John Fastabend <john.fastabend@gmail.com>,
	Tejun Heo <tj@kernel.org>, David Vernet <void@manifault.com>,
	Changwoo Min <changwoo@igalia.com>, Josh Don <joshdon@google.com>,
	Vineeth Pillai <vineethrp@google.com>,
	Peilin Ye <yepeilin@google.com>, Zhi Wang <zhiw@nvidia.com>,
	Lee Trager <ltrager@nvidia.com>,
	Balbir Singh <balbirs@nvidia.com>,
	Shameer Kolothum <skolothumtho@nvidia.com>,
	Himadri Chhaya-Shailesh <himadrics@protonmail.com>,
	NchangRoy <royfru44@gmail.com>,
	bpf@vger.kernel.org, sched-ext@lists.linux.dev,
	linux-kernel@vger.kernel.org
Subject: [PATCH 2/3] bpf: Allow importing device-dax pages into an arena
Date: Tue, 17 Nov 2026 11:28:07 +0100	[thread overview]
Message-ID: <20261117103028.4105609-3-arighi@nvidia.com> (raw)
In-Reply-To: <20261117103028.4105609-1-arighi@nvidia.com>

bpf_arena_import_memfd() covers the host side of a host/guest shared
arena: the host imports a memfd and hands the same file to the VMM. The
guest has no such file. The host memory can be exposed to the guest as
a virtio-pmem or NVDIMM device, which the guest turns into a device-dax
device, /dev/daxX.Y.

Add a sleepable kfunc for syscall programs that imports the device-dax
pages mapped at an address of the calling task:

  int bpf_arena_import_dax(void *map, void __arena *addr,
                           u32 page_cnt, u64 src);

The pages are pinned with pin_user_pages(FOLL_LONGTERM) and mapped like
memfd imports. Only a device-dax mapping is accepted, checked under
mmap_lock together with the pin.

A pin doesn't keep the dev_pagemap of device pages alive, so the import
holds a reference on it until the pages are released: unbinding the
device waits until then. Until the reference is held, an unbind could
free the memmap of the pinned pages, so mem_hotplug_lock is held from
the pin until then.

So removing the device waits for the arena: an unbind blocks until the
pages are freed or the arena is destroyed. Revoking the pages instead
would need a notification from the dax core and a way to tell the BPF
programs that their shared memory is gone, rather than remapping it
under them.

Signed-off-by: Andrea Righi <arighi@nvidia.com>
---
 kernel/bpf/arena.c | 137 ++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 129 insertions(+), 8 deletions(-)

diff --git a/kernel/bpf/arena.c b/kernel/bpf/arena.c
index 799a8cf1ee728..8635bba7907ac 100644
--- a/kernel/bpf/arena.c
+++ b/kernel/bpf/arena.c
@@ -7,7 +7,10 @@
 #include <linux/file.h>
 #include <linux/interval_tree_generic.h>
 #include <linux/irq_work.h>
+#include <linux/magic.h>
 #include <linux/memfd.h>
+#include <linux/memory_hotplug.h>
+#include <linux/memremap.h>
 #include "linux/filter.h"
 #include <linux/llist.h>
 #include <linux/mman.h>
@@ -130,7 +133,9 @@ struct arena_import {
 	/* on a list while being released */
 	struct list_head head;
 	struct page **pages;
-	/* the file, with write access on its inode */
+	/* device pages: one reference keeps them all, NULL otherwise */
+	struct dev_pagemap *pgmap;
+	/* file pages: the file, with write access on its inode, NULL otherwise */
 	struct file *file;
 	long pgoff;
 	long page_cnt;
@@ -396,8 +401,12 @@ static void arena_release_import(struct arena_import *imp)
 {
 	/* BPF programs write through the kernel mapping, which doesn't track dirty */
 	unpin_user_pages_dirty_lock(imp->pages, imp->page_cnt, true);
-	put_write_access(file_inode(imp->file));
-	fput(imp->file);
+	/* After the pins: the memmap of device pages goes away with it */
+	put_dev_pagemap(imp->pgmap);
+	if (imp->file) {
+		put_write_access(file_inode(imp->file));
+		fput(imp->file);
+	}
 	kvfree(imp->pages);
 	kfree(imp);
 }
@@ -1392,11 +1401,11 @@ static int arena_reserve_pages(struct bpf_arena *arena, long uaddr, u32 page_cnt
 
 /*
  * Map @page_cnt pages that the arena didn't allocate at @uaddr, which must be
- * free. On success the arena keeps @pages, their pins and the write access on
- * @file until it releases them, see struct arena_import.
+ * free. On success the arena keeps @pages, their pins, @pgmap and the write
+ * access on @file until it releases them, see struct arena_import.
  */
 static int arena_map_ext_pages(struct bpf_arena *arena, long uaddr, struct page **pages,
-			       long page_cnt, struct file *file)
+			       long page_cnt, struct dev_pagemap *pgmap, struct file *file)
 {
 	/* The first mmap() may be setting the user range: a non-zero start is final */
 	u64 user_vm_start = READ_ONCE(arena->user_vm_start);
@@ -1421,6 +1430,7 @@ static int arena_map_ext_pages(struct bpf_arena *arena, long uaddr, struct page
 	if (!imp)
 		return -ENOMEM;
 	imp->pages = pages;
+	imp->pgmap = pgmap;
 	imp->file = file;
 	imp->pgoff = pgoff;
 	imp->page_cnt = page_cnt;
@@ -1438,7 +1448,8 @@ static int arena_map_ext_pages(struct bpf_arena *arena, long uaddr, struct page
 		WARN_ON_ONCE(apply_to_page_range(&init_mm, kaddr, page_cnt << PAGE_SHIFT,
 						 apply_range_set_cb, &data));
 		flush_vmap_cache(kaddr, page_cnt << PAGE_SHIFT);
-		get_file(file);
+		if (file)
+			get_file(file);
 		arena_import_it_insert(imp, &arena->imports);
 	}
 	raw_res_spin_unlock_irqrestore(&arena->spinlock, flags);
@@ -1513,7 +1524,8 @@ static int arena_import_file(struct bpf_arena *arena, long uaddr, u32 page_cnt,
 		}
 	}
 
-	ret = cnt == page_cnt ? arena_map_ext_pages(arena, uaddr, pages, page_cnt, file) : -EINVAL;
+	ret = cnt == page_cnt ? arena_map_ext_pages(arena, uaddr, pages, page_cnt, NULL, file) :
+	      -EINVAL;
 out:
 	if (ret) {
 		unpin_user_pages(pages, cnt);
@@ -1544,6 +1556,82 @@ static int arena_import_memfd(struct bpf_arena *arena, long uaddr, u32 page_cnt,
 	return arena_import_file(arena, uaddr, page_cnt, fd_file(f), off);
 }
 
+/*
+ * A device-dax file maps the inode of its dax_device, which lives on the dax
+ * pseudo filesystem, whatever CONFIG_FS_DAX says (S_DAX is 0 without it).
+ */
+static bool vma_is_devdax(struct vm_area_struct *vma)
+{
+	struct file *file = vma->vm_file;
+
+	return file && file->f_mapping->host->i_sb->s_magic == DAXFS_MAGIC;
+}
+
+/*
+ * Import the device-dax pages mapped at [src, src + page_cnt * PAGE_SIZE) in
+ * the calling task. Only device-dax is accepted: other drivers hand out
+ * MEMORY_DEVICE_GENERIC pages too (e.g. Xen grants).
+ */
+static int arena_import_dax(struct bpf_arena *arena, long uaddr, u32 page_cnt, u64 src)
+{
+	struct mm_struct *mm = current->mm;
+	struct dev_pagemap *pgmap = NULL;
+	struct vm_area_struct *vma;
+	long i, pinned = 0;
+	struct page **pages;
+	int ret = -EINVAL;
+
+	/* A sleepable callback, like a bpf_wq one, may run without an address space */
+	if (!mm || (src & ~PAGE_MASK))
+		return -EINVAL;
+
+	pages = kvmalloc_array(page_cnt, sizeof(*pages), GFP_KERNEL_ACCOUNT);
+	if (!pages)
+		return -ENOMEM;
+
+	/*
+	 * A pin doesn't keep the memmap of device pages: an unbind can free it
+	 * until a dev_pagemap reference is held, but only under mem_hotplug_lock,
+	 * which nests outside of mmap_lock
+	 */
+	get_online_mems();
+	mmap_read_lock(mm);
+	vma = vma_lookup(mm, src);
+	if (!vma || !vma_is_devdax(vma) ||
+	    ((unsigned long)page_cnt << PAGE_SHIFT) > vma->vm_end - src)
+		goto out_unlock;
+	pinned = pin_user_pages(src, page_cnt, FOLL_WRITE | FOLL_LONGTERM, pages);
+	if (pinned < 0)
+		ret = pinned;
+	else if (pinned == page_cnt && is_zone_device_page(pages[0]))
+		pgmap = get_dev_pagemap(page_to_pfn(pages[0]));
+	if (pgmap && pgmap->type == MEMORY_DEVICE_GENERIC) {
+		ret = 0;
+		/* All of the pages are on the device of the mapping */
+		for (i = 1; i < pinned; i++)
+			if (!is_zone_device_page(pages[i]) || page_pgmap(pages[i]) != pgmap)
+				ret = -EINVAL;
+	}
+	/* Still under the lock: there may be no dev_pagemap reference */
+	if (ret && pinned > 0) {
+		unpin_user_pages(pages, pinned);
+		pinned = 0;
+	}
+out_unlock:
+	mmap_read_unlock(mm);
+	put_online_mems();
+
+	if (!ret)
+		ret = arena_map_ext_pages(arena, uaddr, pages, page_cnt, pgmap, NULL);
+	if (ret) {
+		if (pinned > 0)
+			unpin_user_pages(pages, pinned);
+		put_dev_pagemap(pgmap);
+		kvfree(pages);
+	}
+	return ret;
+}
+
 static void arena_free_worker(struct work_struct *work)
 {
 	struct bpf_arena *arena = container_of(work, struct bpf_arena, free_work);
@@ -1677,6 +1765,38 @@ __bpf_kfunc int bpf_arena_import_memfd(void *p__map, void *addr__ign, u32 page_c
 		return -EINVAL;
 	return arena_import_memfd(arena, (long)addr__ign, page_cnt, fd, off);
 }
+
+/**
+ * bpf_arena_import_dax - map device-dax pages of the calling task into an arena
+ * @p__map: the arena
+ * @addr__ign: page-aligned arena address where the pages are mapped
+ * @page_cnt: number of pages to import
+ * @src: page-aligned address of a writable device-dax mapping in the calling task
+ *
+ * Pin the pages backing @src and map them at @addr__ign, like
+ * bpf_arena_import_memfd() does with the pages of a file. The whole range must
+ * be in one mapping of a device-dax device, e.g. the /dev/daxX.Y that a guest
+ * gets from a virtio-pmem or NVDIMM device backed by host memory, which the
+ * host imports by file. User space keeps accessing the pages through @src.
+ *
+ * Imported pages keep the device in use: unbinding or removing it blocks until
+ * the arena releases them, for as long as a map fd or a bpffs pin keeps the
+ * arena alive.
+ *
+ * BPF programs have no machine check recovery: on a real NVDIMM, the supplier
+ * has to keep bad pages out of the arena.
+ *
+ * Return: 0 on success, negative error code otherwise.
+ */
+__bpf_kfunc int bpf_arena_import_dax(void *p__map, void *addr__ign, u32 page_cnt, u64 src)
+{
+	struct bpf_map *map = p__map;
+	struct bpf_arena *arena = container_of(map, struct bpf_arena, map);
+
+	if (map->map_type != BPF_MAP_TYPE_ARENA || !page_cnt || page_cnt > map->max_entries)
+		return -EINVAL;
+	return arena_import_dax(arena, (long)addr__ign, page_cnt, src);
+}
 __bpf_kfunc_end_defs();
 
 BTF_KFUNCS_START(arena_kfuncs)
@@ -1692,6 +1812,7 @@ static const struct btf_kfunc_id_set common_kfunc_set = {
 
 BTF_KFUNCS_START(arena_syscall_kfuncs)
 BTF_ID_FLAGS(func, bpf_arena_import_memfd, KF_ARENA_ARG2 | KF_SLEEPABLE)
+BTF_ID_FLAGS(func, bpf_arena_import_dax, KF_ARENA_ARG2 | KF_SLEEPABLE)
 BTF_KFUNCS_END(arena_syscall_kfuncs)
 
 static const struct btf_kfunc_id_set syscall_kfunc_set = {
-- 
2.56.1


  parent reply	other threads:[~2026-11-17 10:31 UTC|newest]

Thread overview: 6+ messages / expand[flat|nested]  mbox.gz  Atom feed  top
2026-11-17 10:28 [PATCH v2 bpf-next 0/3] bpf: Import external " Andrea Righi
2026-11-17 10:28 ` [PATCH 1/3] bpf: Allow importing memfd " Andrea Righi
2026-11-17 11:14   ` bot+bpf-ci
2026-11-17 10:28 ` Andrea Righi [this message]
2026-11-17 11:02   ` [PATCH 2/3] bpf: Allow importing device-dax " bot+bpf-ci
2026-11-17 10:28 ` [PATCH 3/3] selftests/bpf: Test importing " Andrea Righi

Reply instructions:

You may reply publicly to this message via plain-text email
using any one of the following methods:

* Save the following mbox file, import it into your mail client,
  and reply-to-all from there: mbox

  Avoid top-posting and favor interleaved quoting:
  https://en.wikipedia.org/wiki/Posting_style#Interleaved_style

* Reply using the --to, --cc, and --in-reply-to
  switches of git-send-email(1):

  git send-email \
    --in-reply-to=20261117103028.4105609-3-arighi@nvidia.com \
    --to=arighi@nvidia.com \
    --cc=andrii@kernel.org \
    --cc=ast@kernel.org \
    --cc=balbirs@nvidia.com \
    --cc=bpf@vger.kernel.org \
    --cc=changwoo@igalia.com \
    --cc=daniel@iogearbox.net \
    --cc=eddyz87@gmail.com \
    --cc=emil@etsalapatis.com \
    --cc=himadrics@protonmail.com \
    --cc=ihor.solodrai@linux.dev \
    --cc=john.fastabend@gmail.com \
    --cc=jolsa@kernel.org \
    --cc=joshdon@google.com \
    --cc=linux-kernel@vger.kernel.org \
    --cc=ltrager@nvidia.com \
    --cc=martin.lau@linux.dev \
    --cc=memxor@gmail.com \
    --cc=royfru44@gmail.com \
    --cc=sched-ext@lists.linux.dev \
    --cc=skolothumtho@nvidia.com \
    --cc=song@kernel.org \
    --cc=tj@kernel.org \
    --cc=vineethrp@google.com \
    --cc=void@manifault.com \
    --cc=yepeilin@google.com \
    --cc=yonghong.song@linux.dev \
    --cc=zhiw@nvidia.com \
    /path/to/YOUR_REPLY

  https://kernel.org/pub/software/scm/git/docs/git-send-email.html

* If your mail client supports setting the In-Reply-To header
  via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox

all inboxes | Powered by JetHome®