* [PATCH bpf-next v12 2/2] selftests/bpf: Add memcg async reclaim test
2026-09-18 6:58 [PATCH bpf-next v12 0/2] bpf: BPF-driven proactive memcg reclaim Hui Zhu
2026-09-18 6:58 ` [PATCH bpf-next v12 1/2] mm/bpf: Add bpf_proactive_reclaim kfunc Hui Zhu
@ 2026-09-18 6:58 ` Hui Zhu
2026-09-18 8:07 ` bot+bpf-ci
2026-09-18 15:41 ` Alexei Starovoitov
1 sibling, 2 replies; 8+ messages in thread
From: Hui Zhu @ 2026-09-18 6:58 UTC (permalink / raw)
To: Roman Gushchin, JP Kobryn, Shakeel Butt, Andrew Morton,
Andrii Nakryiko, Eduard Zingerman, Ihor Solodrai,
Alexei Starovoitov, Daniel Borkmann, Kumar Kartikeya Dwivedi,
Martin KaFai Lau, Song Liu, Yonghong Song, Jiri Olsa,
Emil Tsalapatis, Shuah Khan, David Hildenbrand, Barry Song,
Geliang Tang, linux-kernel, bpf, linux-mm, linux-kselftest
Cc: Hui Zhu
From: Hui Zhu <zhuhui@kylinos.cn>
Exercise BPF-driven async proactive reclaim. A BPF program watches the
refault stats of a memory-pressured cgroup and, once they grow, reclaims
a second cgroup from a bpf_wq callback through bpf_proactive_reclaim().
Pass/fail rests on the reclaim counters. The workload timings are a
benchmark, not a criterion, so they are logged only under
TEST_MEMCG_ASYNC_RECLAIM_BENCH=1, which also runs a baseline without the
BPF program, recreating the cgroups in between so both runs start cold.
A second test removes the reclaim target while the program keeps running
and verifies that reclaim stops. The program skips a target whose css has
lost CSS_ONLINE, but that window sits inside rmdir() and is too narrow to
hit reliably, so the test asserts the post-rmdir outcome and only counts
the in-window one.
Add CONFIG_MEMCG to the selftests config fragment because
bpf_proactive_reclaim() is built only under it and the program fails to
load without it.
Signed-off-by: Hui Zhu <zhuhui@kylinos.cn>
---
tools/testing/selftests/bpf/config | 1 +
.../bpf/prog_tests/memcg_async_reclaim.c | 890 ++++++++++++++++++
.../selftests/bpf/progs/memcg_async_reclaim.c | 327 +++++++
3 files changed, 1218 insertions(+)
create mode 100644 tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
create mode 100644 tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
diff --git a/tools/testing/selftests/bpf/config b/tools/testing/selftests/bpf/config
index 2f79688dcf7c..1edaa752facb 100644
--- a/tools/testing/selftests/bpf/config
+++ b/tools/testing/selftests/bpf/config
@@ -57,6 +57,7 @@ CONFIG_LIRC=y
CONFIG_LIVEPATCH=y
CONFIG_LWTUNNEL=y
CONFIG_LWTUNNEL_BPF=y
+CONFIG_MEMCG=y
CONFIG_MODULE_SIG=y
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_MODULE_UNLOAD=y
diff --git a/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
new file mode 100644
index 000000000000..179b64fb6de8
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/memcg_async_reclaim.c
@@ -0,0 +1,890 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Memory controller eBPF async reclaim test
+ *
+ * Setting TEST_MEMCG_ASYNC_RECLAIM_BENCH=1 adds a baseline run of the
+ * workload without the BPF program to memcg_async_reclaim, and prints
+ * the baseline and reclaim timings, plus the speedup of the pressured
+ * cgroup, to stdout. The cgroups are recreated between the two runs so
+ * that both start from the same cold state.
+ */
+
+#include <test_progs.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/vfs.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <time.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <limits.h>
+#include <linux/magic.h>
+
+#include "cgroup_helpers.h"
+
+struct bpf_args {
+ u64 high_cgroup_id;
+ u64 low_cgroup_id;
+ u64 event_delta_threshold;
+ u64 check_ns;
+};
+
+/* Reclaim outcomes reported by the BPF program through the ringbuf. */
+enum reclaim_outcome {
+ RECLAIM_OUTCOME_CALLED,
+ RECLAIM_OUTCOME_SKIPPED_DYING,
+ RECLAIM_OUTCOME_TARGET_GONE,
+};
+
+struct reclaim_event {
+ u64 outcome;
+};
+
+#include "memcg_async_reclaim.skel.h"
+
+#define FILE_SIZE (32 * 1024 * 1024ul)
+#define BUFFER_SIZE (4096)
+#define CG_LIMIT (32 * 1024 * 1024ul)
+#define READ_TIMES 50
+
+#define CG_DIR "/memcg_async_reclaim"
+#define CG_HIGH_DIR CG_DIR "/high"
+#define CG_LOW_DIR CG_DIR "/low"
+
+#define CG_DYING_DIR "/memcg_async_reclaim_dying"
+#define CG_DYING_TRIGGER_DIR CG_DYING_DIR "/trigger"
+#define CG_DYING_TARGET_DIR CG_DYING_DIR "/target"
+
+#define CHECK_PERIOD_NS (2 * 1000 * 1000ull)
+#define EVENT_DELTA_THRESHOLD 1
+
+/*
+ * Timing for the dying test: after the target cgroup is removed, give
+ * in-flight reclaim passes time to drain, then wait for a reclaim round
+ * to hit the removed target. The keepalive reader keeps the trigger
+ * cgroup refaulting, and the timer fires every CHECK_PERIOD_NS, so
+ * such a round must show up within a few timer periods. The BPF
+ * program reports every reclaim outcome through the ringbuf, so the
+ * waits just poll it with a timeout.
+ */
+#define DYING_SETTLE_US (200 * 1000)
+#define EVENT_POLL_TIMEOUT_MS (100)
+#define EVENT_POLL_ROUNDS (50)
+
+static bool usable_for_workload_files(const char *dir)
+{
+ struct statfs st;
+
+ if (statfs(dir, &st))
+ return false;
+
+ return st.f_type != TMPFS_MAGIC && st.f_type != RAMFS_MAGIC;
+}
+
+/*
+ * The workload files must sit on a regular filesystem: with swap
+ * disabled for the cgroup, tmpfs/ramfs pages are unevictable and would
+ * OOM the cgroup instead of exercising reclaim; they are also charged
+ * as anonymous memory, so they never raise the WORKINGSET_REFAULT_FILE
+ * events the BPF program monitors.
+ *
+ * /tmp is tmpfs on many systems, so fall back to the current directory.
+ * That fallback needs the very same check: test_progs is routinely run
+ * from a tmpfs working directory, and silently landing there produces an
+ * OOM that looks like a reclaim failure. Returns NULL when neither
+ * directory is usable, in which case the workload cannot run at all.
+ */
+static const char *workload_files_dir(void)
+{
+ if (usable_for_workload_files("/tmp"))
+ return "/tmp";
+ if (usable_for_workload_files("."))
+ return ".";
+
+ return NULL;
+}
+
+/*
+ * Keep the workload from swapping out, so that reclaim has to evict page
+ * cache instead. memory.swap.max only exists when the kernel is built with
+ * CONFIG_SWAP, and /proc/swaps is registered by the same CONFIG_SWAP-gated
+ * code (mm/swapfile.c), so it stands in for the file here. With CONFIG_SWAP=n
+ * no swap can happen anyway, so skipping the write is the correct behaviour.
+ */
+static int disable_swap(const char *cgroup_dir)
+{
+ if (access("/proc/swaps", F_OK))
+ return 0;
+
+ return write_cgroup_file(cgroup_dir, "memory.swap.max", "0");
+}
+
+/*
+ * The forked children run after test_progs hijacked stdio, so anything
+ * they print is lost with their private copy of the hijacked buffer
+ * unless the test runs under -v. The exit status is the one diagnostics
+ * channel that always reaches the parent, so each failing step gets its
+ * own code.
+ */
+enum child_exit_code {
+ CHILD_EXIT_OK = 0,
+ CHILD_EXIT_JOIN_CGROUP,
+ CHILD_EXIT_WRITE_FILE,
+ CHILD_EXIT_READ_FILE,
+ CHILD_EXIT_TIME_FILE,
+};
+
+static const char *child_exit_str(int code)
+{
+ switch (code) {
+ case CHILD_EXIT_OK:
+ return "success";
+ case CHILD_EXIT_JOIN_CGROUP:
+ return "join cgroup";
+ case CHILD_EXIT_WRITE_FILE:
+ return "write data file";
+ case CHILD_EXIT_READ_FILE:
+ return "read data file";
+ case CHILD_EXIT_TIME_FILE:
+ return "write time file";
+ default:
+ return "unknown";
+ }
+}
+
+static int setup_high_low_cgroups(u64 *high_cgroup_id, u64 *low_cgroup_id)
+{
+ int ret;
+ char limit_buf[20];
+
+ ret = setup_cgroup_environment();
+ if (!ASSERT_OK(ret, "setup_cgroup_environment"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DIR))
+ goto cleanup;
+ close(ret);
+
+ ret = enable_controllers(CG_DIR, "memory");
+ if (!ASSERT_OK(ret, "enable_controllers"))
+ goto cleanup;
+
+ snprintf(limit_buf, sizeof(limit_buf), "%lu", CG_LIMIT);
+ ret = write_cgroup_file(CG_DIR, "memory.max", limit_buf);
+ if (!ASSERT_OK(ret, "write_cgroup_file memory.max"))
+ goto cleanup;
+
+ ret = disable_swap(CG_DIR);
+ if (!ASSERT_OK(ret, "disable_swap"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_HIGH_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_HIGH_DIR))
+ goto cleanup;
+ close(ret);
+
+ *high_cgroup_id = get_cgroup_id(CG_HIGH_DIR);
+ if (!ASSERT_GT(*high_cgroup_id, 0, "get_cgroup_id"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_LOW_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_LOW_DIR))
+ goto cleanup;
+ close(ret);
+
+ *low_cgroup_id = get_cgroup_id(CG_LOW_DIR);
+ if (!ASSERT_GT(*low_cgroup_id, 0, "get_cgroup_id"))
+ goto cleanup;
+
+ return 0;
+
+cleanup:
+ cleanup_cgroup_environment();
+ return -1;
+}
+
+/*
+ * The dying test needs an empty reclaim target plus a cgroup that keeps
+ * refaulting while the target is removed, so reclaim rounds keep
+ * starting and run into the removed target. The two have to be separate
+ * cgroups: the target must hold no processes to be removed, and v2's
+ * no-internal-process constraint keeps the refaulting workload out of
+ * any parent that has domain children.
+ */
+static int setup_dying_cgroups(u64 *trigger_cgroup_id, u64 *target_cgroup_id)
+{
+ int ret;
+ char limit_buf[20];
+
+ ret = setup_cgroup_environment();
+ if (!ASSERT_OK(ret, "setup_cgroup_environment"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_DYING_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DYING_DIR))
+ goto cleanup;
+ close(ret);
+
+ ret = enable_controllers(CG_DYING_DIR, "memory");
+ if (!ASSERT_OK(ret, "enable_controllers"))
+ goto cleanup;
+
+ snprintf(limit_buf, sizeof(limit_buf), "%lu", CG_LIMIT);
+ ret = write_cgroup_file(CG_DYING_DIR, "memory.max", limit_buf);
+ if (!ASSERT_OK(ret, "write_cgroup_file memory.max"))
+ goto cleanup;
+
+ ret = disable_swap(CG_DYING_DIR);
+ if (!ASSERT_OK(ret, "disable_swap"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_DYING_TRIGGER_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DYING_TRIGGER_DIR))
+ goto cleanup;
+ close(ret);
+
+ *trigger_cgroup_id = get_cgroup_id(CG_DYING_TRIGGER_DIR);
+ if (!ASSERT_GT(*trigger_cgroup_id, 0, "get_cgroup_id"))
+ goto cleanup;
+
+ ret = create_and_get_cgroup(CG_DYING_TARGET_DIR);
+ if (!ASSERT_GE(ret, 0, "create_and_get_cgroup " CG_DYING_TARGET_DIR))
+ goto cleanup;
+ close(ret);
+
+ *target_cgroup_id = get_cgroup_id(CG_DYING_TARGET_DIR);
+ if (!ASSERT_GT(*target_cgroup_id, 0, "get_cgroup_id"))
+ goto cleanup;
+
+ return 0;
+
+cleanup:
+ cleanup_cgroup_environment();
+ return -1;
+}
+
+static int write_file(int fd)
+{
+ int ret = -1;
+ size_t written = 0;
+ char *buffer;
+
+ buffer = malloc(BUFFER_SIZE);
+ if (!buffer)
+ goto out;
+
+ memset(buffer, 'A', BUFFER_SIZE);
+
+ while (written < FILE_SIZE) {
+ size_t to_write = FILE_SIZE - written < BUFFER_SIZE ?
+ FILE_SIZE - written : BUFFER_SIZE;
+ ssize_t n = write(fd, buffer, to_write);
+
+ if (n <= 0)
+ goto cleanup;
+ written += n;
+ }
+
+ ret = 0;
+cleanup:
+ free(buffer);
+out:
+ return ret;
+}
+
+static int read_file(int fd, int iterations)
+{
+ long page_size = sysconf(_SC_PAGESIZE);
+ struct stat sb;
+ char *map;
+ size_t i;
+
+ if (fstat(fd, &sb) == -1)
+ return -1;
+
+ if (sb.st_size != FILE_SIZE) {
+ fprintf(stderr, "File size mismatch: expected %lu, got %lu\n",
+ (unsigned long)FILE_SIZE, (unsigned long)sb.st_size);
+ return -1;
+ }
+
+ map = mmap(NULL, FILE_SIZE, PROT_READ, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ return -1;
+
+ for (int iter = 0; iter < iterations; iter++) {
+ for (i = 0; i < FILE_SIZE; i += page_size) {
+ /* access a byte to trigger page fault */
+ volatile char v = map[i];
+ (void)v;
+ }
+ }
+
+ if (munmap(map, FILE_SIZE) == -1)
+ return -1;
+
+ return 0;
+}
+
+static int real_test_child_work(const char *cgroup_path, int data_fd,
+ int time_fd, int read_times)
+{
+ struct timespec start, end;
+ double elapsed;
+ char buf[32];
+ int len;
+
+ if (join_parent_cgroup(cgroup_path))
+ return CHILD_EXIT_JOIN_CGROUP;
+
+ clock_gettime(CLOCK_MONOTONIC, &start);
+
+ if (write_file(data_fd))
+ return CHILD_EXIT_WRITE_FILE;
+
+ if (read_file(data_fd, read_times))
+ return CHILD_EXIT_READ_FILE;
+
+ clock_gettime(CLOCK_MONOTONIC, &end);
+
+ elapsed = (end.tv_sec - start.tv_sec) +
+ (end.tv_nsec - start.tv_nsec) / 1000000000.0;
+
+ len = snprintf(buf, sizeof(buf), "%.6f", elapsed);
+ /* snprintf() returns the untruncated length, so bound it before write() */
+ if (len < 0 || len >= (int)sizeof(buf) || write(time_fd, buf, len) != len)
+ return CHILD_EXIT_TIME_FILE;
+
+ return CHILD_EXIT_OK;
+}
+
+static int get_time(int fd, double *time)
+{
+ char buf[64] = {};
+ ssize_t n;
+
+ /* The child wrote through the fork-shared description, so rewind. */
+ if (lseek(fd, 0, SEEK_SET) == (off_t)-1) {
+ PRINT_FAIL("lseek time file");
+ return -1;
+ }
+
+ n = read(fd, buf, sizeof(buf) - 1);
+ if (!ASSERT_GT(n, 0, "read time file"))
+ return -1;
+
+ if (sscanf(buf, "%lf", time) != 1) {
+ PRINT_FAIL("sscanf %s", buf);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+run_high_low_workload(const char *dir, double *high_elapsed, double *low_elapsed,
+ int read_times)
+{
+ char high_data_file[PATH_MAX];
+ char low_data_file[PATH_MAX];
+ char high_time_file[PATH_MAX];
+ char low_time_file[PATH_MAX];
+ int high_data_fd = -1, low_data_fd = -1;
+ int high_time_fd = -1, low_time_fd = -1;
+ pid_t high_pid = -1, low_pid = -1;
+ pid_t wait_ret;
+ int status;
+ int ret = -1;
+
+ snprintf(high_data_file, sizeof(high_data_file),
+ "%s/memcg_async_high_data_XXXXXX", dir);
+ snprintf(low_data_file, sizeof(low_data_file),
+ "%s/memcg_async_low_data_XXXXXX", dir);
+ snprintf(high_time_file, sizeof(high_time_file),
+ "%s/memcg_async_high_time_XXXXXX", dir);
+ snprintf(low_time_file, sizeof(low_time_file),
+ "%s/memcg_async_low_time_XXXXXX", dir);
+
+ /*
+ * The mkstemp() fds are kept and inherited by the children rather
+ * than reopened by name: reopening would resolve the path again and
+ * follow whatever sits there by then.
+ */
+ high_data_fd = mkstemp(high_data_file);
+ if (!ASSERT_GE(high_data_fd, 0, "mkstemp high data"))
+ goto cleanup;
+
+ low_data_fd = mkstemp(low_data_file);
+ if (!ASSERT_GE(low_data_fd, 0, "mkstemp low data"))
+ goto cleanup;
+
+ high_time_fd = mkstemp(high_time_file);
+ if (!ASSERT_GE(high_time_fd, 0, "mkstemp high time"))
+ goto cleanup;
+
+ low_time_fd = mkstemp(low_time_file);
+ if (!ASSERT_GE(low_time_fd, 0, "mkstemp low time"))
+ goto cleanup;
+
+ low_pid = fork();
+ if (!ASSERT_GE(low_pid, 0, "fork low"))
+ goto cleanup;
+ if (low_pid == 0)
+ _exit(real_test_child_work(CG_LOW_DIR, low_data_fd,
+ low_time_fd, read_times));
+
+ high_pid = fork();
+ if (!ASSERT_GE(high_pid, 0, "fork high"))
+ goto cleanup;
+ if (high_pid == 0)
+ _exit(real_test_child_work(CG_HIGH_DIR, high_data_fd,
+ high_time_fd, read_times));
+
+ wait_ret = waitpid(low_pid, &status, 0);
+ if (!ASSERT_GT(wait_ret, 0, "low waitpid"))
+ goto cleanup;
+ /*
+ * The child has been reaped and its PID can already be reused,
+ * so mark it to keep cleanup from signaling an unrelated process.
+ */
+ low_pid = -1;
+ if (!ASSERT_TRUE(WIFEXITED(status), "low exited"))
+ goto cleanup;
+ if (WEXITSTATUS(status) != CHILD_EXIT_OK) {
+ PRINT_FAIL("low child failed at: %s (exit status %d)",
+ child_exit_str(WEXITSTATUS(status)),
+ WEXITSTATUS(status));
+ goto cleanup;
+ }
+
+ wait_ret = waitpid(high_pid, &status, 0);
+ if (!ASSERT_GT(wait_ret, 0, "high waitpid"))
+ goto cleanup;
+ /* Same as above: the reaped PID must not be signaled again. */
+ high_pid = -1;
+ if (!ASSERT_TRUE(WIFEXITED(status), "high exited"))
+ goto cleanup;
+ if (WEXITSTATUS(status) != CHILD_EXIT_OK) {
+ PRINT_FAIL("high child failed at: %s (exit status %d)",
+ child_exit_str(WEXITSTATUS(status)),
+ WEXITSTATUS(status));
+ goto cleanup;
+ }
+
+ if (get_time(high_time_fd, high_elapsed))
+ goto cleanup;
+ if (get_time(low_time_fd, low_elapsed))
+ goto cleanup;
+
+ ret = 0;
+
+cleanup:
+ /* On failure, make sure no child process is left behind */
+ if (ret) {
+ if (high_pid > 0) {
+ kill(high_pid, SIGKILL);
+ (void)waitpid(high_pid, NULL, 0);
+ }
+ if (low_pid > 0) {
+ kill(low_pid, SIGKILL);
+ (void)waitpid(low_pid, NULL, 0);
+ }
+ }
+ if (high_data_fd >= 0)
+ close(high_data_fd);
+ if (low_data_fd >= 0)
+ close(low_data_fd);
+ if (high_time_fd >= 0)
+ close(high_time_fd);
+ if (low_time_fd >= 0)
+ close(low_time_fd);
+ unlink(low_time_file);
+ unlink(high_time_file);
+ unlink(low_data_file);
+ unlink(high_data_file);
+ return ret;
+}
+
+static int
+setup_bpf(u64 high_cgroup_id, u64 low_cgroup_id,
+ struct memcg_async_reclaim **skel_ptr)
+{
+ struct memcg_async_reclaim *skel;
+ struct bpf_args args = {
+ .high_cgroup_id = high_cgroup_id,
+ .low_cgroup_id = low_cgroup_id,
+ .event_delta_threshold = EVENT_DELTA_THRESHOLD,
+ .check_ns = CHECK_PERIOD_NS,
+ };
+ LIBBPF_OPTS(bpf_test_run_opts, run_opts,
+ .ctx_in = &args,
+ .ctx_size_in = sizeof(args));
+ int prog_init_fd, err;
+
+ skel = memcg_async_reclaim__open_and_load();
+ if (!ASSERT_OK_PTR(skel, "memcg_async_reclaim__open_and_load"))
+ return -1;
+
+ prog_init_fd = bpf_program__fd(skel->progs.wq_prog_init);
+
+ err = bpf_prog_test_run_opts(prog_init_fd, &run_opts);
+ if (!ASSERT_OK(err, "bpf_prog_test_run_opts"))
+ goto error_out;
+ if (!ASSERT_EQ(run_opts.retval, 0, "prog_init retval"))
+ goto error_out;
+
+ *skel_ptr = skel;
+ return 0;
+
+error_out:
+ memcg_async_reclaim__destroy(skel);
+ return -1;
+}
+
+/*
+ * A timer that stopped rearming produces no further reclaim rounds, which
+ * the "no events" failures below would otherwise report as a kernel bug.
+ * Only call this on a path that is already failing.
+ */
+static void report_timer_failures(struct memcg_async_reclaim *skel)
+{
+ u64 failures = skel->bss->timer_failures;
+
+ if (failures)
+ PRINT_FAIL("bpf_timer failed to rearm %llu time(s), so the reclaim loop stopped",
+ (unsigned long long)failures);
+}
+
+/*
+ * A kfunc that fails on every call is indistinguishable from a cgroup with
+ * nothing left to reclaim, which the reclaim_calls and reclaimed_bytes
+ * assertions below cannot tell apart. Only call this on a path that is
+ * already failing.
+ */
+static void report_reclaim_errors(struct memcg_async_reclaim *skel)
+{
+ u64 last_err = skel->bss->last_reclaim_err;
+
+ if (last_err)
+ PRINT_FAIL("bpf_proactive_reclaim() last failed with -%llu, nothing was reclaimed",
+ (unsigned long long)last_err);
+}
+
+/*
+ * The benchmark numbers are only interesting when the test passes, but
+ * dump_test_log() drops the captured log unless the test failed or -v was
+ * given, so they have to bypass the hijack. The ?: stdout fallback is
+ * required, not defensive: stdio_hijack() returns early in verbose mode and
+ * never sets env.stdout_saved.
+ *
+ * This also bypasses the worker-to-dispatcher log protocol, so under -j the
+ * line can interleave with the per-test records. It carries no '#' prefix:
+ * test_progs uses that only for its own result records, and print_test_log()
+ * dumps free-form test output verbatim.
+ */
+static void bench_printf(const char *fmt, ...)
+{
+ va_list ap;
+
+ va_start(ap, fmt);
+ vfprintf(env.stdout_saved ?: stdout, fmt, ap);
+ va_end(ap);
+}
+
+void test_memcg_async_reclaim(void)
+{
+ u64 high_cgroup_id, low_cgroup_id;
+ double high_time = 0.0, low_time = 0.0;
+ double base_high_time = 0.0, base_low_time = 0.0;
+ struct memcg_async_reclaim *skel = NULL;
+ const char *dir;
+ int err, bench;
+
+ dir = workload_files_dir();
+ if (!ASSERT_TRUE(dir != NULL, "workload_files_dir"))
+ return;
+
+ bench = !!getenv("TEST_MEMCG_ASYNC_RECLAIM_BENCH");
+
+ err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id);
+ if (!ASSERT_OK(err, "setup_high_low_cgroups reclaim"))
+ return;
+
+ /*
+ * Optional baseline for the benchmark numbers below: run the
+ * same workload once without the BPF program, so the log can
+ * report how much async reclaim improved the pressured cgroup.
+ */
+ if (bench) {
+ err = run_high_low_workload(dir, &base_high_time,
+ &base_low_time, READ_TIMES);
+ if (!ASSERT_OK(err, "run_high_low_workload baseline"))
+ goto out;
+
+ /*
+ * The baseline left up to CG_LIMIT of page cache charged to the
+ * high and low cgroups, and memcg charges outlive the workload
+ * processes. Recreate the cgroups so that the instrumented run
+ * below starts from the same cold state the baseline did;
+ * otherwise the reported speedup compares a cold run against a
+ * warm one.
+ */
+ cleanup_cgroup_environment();
+ err = setup_high_low_cgroups(&high_cgroup_id, &low_cgroup_id);
+ if (!ASSERT_OK(err, "setup_high_low_cgroups after baseline"))
+ goto out;
+ }
+
+ err = setup_bpf(high_cgroup_id, low_cgroup_id, &skel);
+ if (!ASSERT_OK(err, "setup_bpf"))
+ goto out;
+
+ err = run_high_low_workload(dir, &high_time, &low_time, READ_TIMES);
+ if (!ASSERT_OK(err, "run_high_low_workload reclaim"))
+ goto out;
+
+ /*
+ * Timing alone cannot distinguish a working reclaim from a no-op
+ * one, so require that the BPF program actually reclaimed memory
+ * from the low cgroup.
+ */
+ if (!ASSERT_GT(skel->bss->reclaim_calls, 0, "reclaim_calls")) {
+ report_timer_failures(skel);
+ report_reclaim_errors(skel);
+ goto out;
+ }
+ if (!ASSERT_GT(skel->bss->reclaimed_bytes, 0, "reclaimed_bytes")) {
+ report_reclaim_errors(skel);
+ goto out;
+ }
+
+ /*
+ * The timing comparison is a benchmark and too flaky to fail CI
+ * runs, so it only logs the numbers; the counters above already
+ * proved that the BPF program reclaimed memory.
+ */
+ if (bench && base_high_time > 0.0)
+ bench_printf("memcg_async_reclaim: baseline high=%f low=%f, "
+ "reclaim high=%f low=%f, high speedup=%.1f%%\n",
+ base_high_time, base_low_time, high_time, low_time,
+ 100.0 * (base_high_time - high_time) / base_high_time);
+
+out:
+ if (skel)
+ memcg_async_reclaim__destroy(skel);
+ cleanup_cgroup_environment();
+}
+
+/*
+ * Keep refaults flowing through the trigger cgroup so reclaim rounds
+ * keep being triggered while the target cgroup is being removed. The
+ * child joins the trigger cgroup and writes the data file there, so
+ * that the file pages are charged to the trigger cgroup and actually
+ * come under its memory limit; then it re-reads the file in a loop
+ * until it is killed.
+ */
+static pid_t spawn_keepalive_reader(int data_fd)
+{
+ pid_t pid = fork();
+
+ if (pid != 0)
+ return pid;
+
+ if (join_parent_cgroup(CG_DYING_TRIGGER_DIR))
+ _exit(CHILD_EXIT_JOIN_CGROUP);
+ if (write_file(data_fd))
+ _exit(CHILD_EXIT_WRITE_FILE);
+ for (;;) {
+ if (read_file(data_fd, READ_TIMES))
+ _exit(CHILD_EXIT_READ_FILE);
+ }
+}
+
+/* Reclaim outcomes observed so far, tallied from ringbuf events. */
+struct reclaim_events_seen {
+ int called;
+ int skipped_dying;
+ int target_gone;
+};
+
+static int on_reclaim_event(void *ctx, void *data, size_t sz)
+{
+ struct reclaim_events_seen *seen = ctx;
+ const struct reclaim_event *ev = data;
+
+ if (sz < sizeof(*ev))
+ return 0;
+
+ switch (ev->outcome) {
+ case RECLAIM_OUTCOME_CALLED:
+ seen->called++;
+ break;
+ case RECLAIM_OUTCOME_SKIPPED_DYING:
+ seen->skipped_dying++;
+ break;
+ case RECLAIM_OUTCOME_TARGET_GONE:
+ seen->target_gone++;
+ break;
+ }
+
+ return 0;
+}
+
+/*
+ * Remove the reclaim target while the BPF program keeps running and verify
+ * that reclaim stops instead of reclaiming from the removed cgroup.
+ *
+ * The target stays empty; the workload lives in the trigger cgroup and only
+ * keeps refaults flowing so that reclaim rounds keep starting, both before
+ * and after the target is removed. A CALLED event while the target is alive
+ * proves that rounds really run (the kfunc returns 0 on the empty target, but
+ * the call is still counted).
+ *
+ * Once rmdir has returned, bpf_cgroup_from_id() has already stopped resolving
+ * the target: kernfs deactivates the directory node inside the rmdir syscall,
+ * see cgroup_is_dying() in the BPF program. TARGET_GONE is therefore the
+ * outcome this test can rely on, and the one it asserts. SKIPPED_DYING needs
+ * an attempt to land in the short window between cgroup_destroy_locked()
+ * clearing CSS_ONLINE and kernfs_remove() deactivating the node, so it is
+ * counted and reported on failure but never required.
+ */
+void test_memcg_async_reclaim_dying(void)
+{
+ u64 trigger_cgroup_id, target_cgroup_id;
+ char data_file[PATH_MAX] = "";
+ struct reclaim_events_seen seen = {};
+ struct memcg_async_reclaim *skel = NULL;
+ struct ring_buffer *rb = NULL;
+ u64 calls_before;
+ const char *dir;
+ pid_t reader_pid = -1;
+ int data_fd = -1;
+ int err, status, i, n;
+
+ dir = workload_files_dir();
+ if (!ASSERT_TRUE(dir != NULL, "workload_files_dir"))
+ return;
+
+ err = setup_dying_cgroups(&trigger_cgroup_id, &target_cgroup_id);
+ if (!ASSERT_OK(err, "setup_dying_cgroups"))
+ return;
+
+ err = setup_bpf(trigger_cgroup_id, target_cgroup_id, &skel);
+ if (!ASSERT_OK(err, "setup_bpf"))
+ goto out;
+
+ rb = ring_buffer__new(bpf_map__fd(skel->maps.reclaim_events),
+ on_reclaim_event, &seen, NULL);
+ if (!ASSERT_OK_PTR(rb, "ring_buffer__new"))
+ goto out;
+
+ snprintf(data_file, sizeof(data_file),
+ "%s/memcg_async_dying_XXXXXX", dir);
+ /* Kept open and inherited, see run_high_low_workload(). */
+ data_fd = mkstemp(data_file);
+ if (!ASSERT_GE(data_fd, 0, "mkstemp"))
+ goto out;
+
+ reader_pid = spawn_keepalive_reader(data_fd);
+ if (!ASSERT_GT(reader_pid, 0, "fork keepalive reader"))
+ goto out;
+
+ /* Wait for reclaim rounds to reach the live target cgroup. */
+ for (i = 0; i < EVENT_POLL_ROUNDS && !seen.called; i++) {
+ n = ring_buffer__poll(rb, EVENT_POLL_TIMEOUT_MS);
+ if (!ASSERT_GE(n, 0, "ring_buffer__poll"))
+ goto out;
+
+ /*
+ * The reader loops until killed, so finding it already gone
+ * means the refaults the BPF program waits for never started.
+ * Report its exit code rather than letting the assert below
+ * blame the kernel for the full poll window.
+ */
+ if (waitpid(reader_pid, &status, WNOHANG) == reader_pid) {
+ /* Reaped, so cleanup must not signal a reused PID. */
+ reader_pid = -1;
+ if (WIFEXITED(status))
+ PRINT_FAIL("keepalive reader exited early: %s",
+ child_exit_str(WEXITSTATUS(status)));
+ else
+ PRINT_FAIL("keepalive reader died, status 0x%x",
+ status);
+ goto out;
+ }
+ }
+ if (!ASSERT_GT(seen.called, 0, "reclaim events")) {
+ report_timer_failures(skel);
+ goto out;
+ }
+
+ remove_cgroup(CG_DYING_TARGET_DIR);
+
+ /* Let reclaim passes that were already in flight drain. */
+ usleep(DYING_SETTLE_US);
+
+ calls_before = skel->bss->reclaim_calls;
+
+ /*
+ * Wait for a reclaim round to hit the removed cgroup. Only TARGET_GONE
+ * ends the wait: an attempt that lands inside the rmdir window reports
+ * SKIPPED_DYING first, and stopping there would miss the events that
+ * follow it.
+ */
+ for (i = 0; i < EVENT_POLL_ROUNDS && !seen.target_gone; i++) {
+ n = ring_buffer__poll(rb, EVENT_POLL_TIMEOUT_MS);
+ if (!ASSERT_GE(n, 0, "ring_buffer__poll"))
+ goto out;
+ }
+
+ /*
+ * TARGET_GONE is what the test can rely on: by the time rmdir has
+ * returned, bpf_cgroup_from_id() has stopped resolving the target.
+ * SKIPPED_DYING only happens if an attempt lands inside the rmdir
+ * window, so it is reported here rather than asserted.
+ */
+ if (!seen.target_gone) {
+ report_timer_failures(skel);
+ PRINT_FAIL("no reclaim round hit the removed cgroup (gone=%d, dying=%d)",
+ seen.target_gone, seen.skipped_dying);
+ goto out;
+ }
+
+ /*
+ * Reclaim must have stopped with the target. Both counters stay put for
+ * the same reason: once rmdir has returned, bpf_cgroup_from_id() fails,
+ * so every attempt takes the TARGET_GONE path before it can reach
+ * reclaim_calls++. This therefore checks that reclaim really stops, not
+ * that cgroup_is_dying() works -- see that function in the BPF program
+ * for why the dying window cannot be hit reliably from userspace.
+ *
+ * reclaimed_bytes is compared against 0 rather than against its
+ * pre-removal value: the target is empty for the whole test, so
+ * bpf_proactive_reclaim() has nothing to reclaim and must have returned
+ * 0 on every call, before and after the removal alike. An "unchanged"
+ * comparison would be 0 == 0 and could never fail.
+ */
+ if (!ASSERT_EQ(skel->bss->reclaim_calls, calls_before, "reclaim_calls"))
+ goto out;
+ if (!ASSERT_EQ(skel->bss->reclaimed_bytes, 0ULL, "reclaimed_bytes"))
+ goto out;
+
+out:
+ if (reader_pid > 0) {
+ kill(reader_pid, SIGKILL);
+ (void)waitpid(reader_pid, NULL, 0);
+ }
+ if (data_fd >= 0)
+ close(data_fd);
+ if (data_file[0])
+ unlink(data_file);
+ if (rb)
+ ring_buffer__free(rb);
+ if (skel)
+ memcg_async_reclaim__destroy(skel);
+ cleanup_cgroup_environment();
+}
diff --git a/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
new file mode 100644
index 000000000000..d78d547af7bf
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/memcg_async_reclaim.c
@@ -0,0 +1,327 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "vmlinux.h"
+#include "bpf_experimental.h"
+
+#define CLOCK_MONOTONIC 1
+/*
+ * One reclaim round targets RECLAIM_MAX_ITER batches of RECLAIM_SIZE
+ * each, i.e. 4 MiB in total. Each bpf_wq callback reclaims a single
+ * batch and requeues the same work item for the next one, so no callback
+ * runs longer than one bounded reclaim pass.
+ *
+ * RECLAIM_SIZE is a byte count, not a page count: bpf_proactive_reclaim()
+ * converts it against the running kernel's PAGE_SIZE and returns bytes, so
+ * this holds on any page size. The kfunc also clamps a batch to the
+ * kernel's MEMCG_CHARGE_BATCH, which is not visible to BPF; a clamped
+ * batch only means a round needs more callbacks, because remaining is
+ * decremented by the bytes actually reclaimed.
+ */
+#define RECLAIM_SIZE (128 * 1024UL)
+#define RECLAIM_MAX_ITER 32
+
+struct bpf_args {
+ u64 high_cgroup_id;
+ u64 low_cgroup_id;
+ u64 event_delta_threshold;
+ u64 check_ns;
+};
+
+struct cgroup_memcg {
+ struct cgroup *cgrp;
+ struct mem_cgroup *memcg;
+};
+
+static u64 wq_high_cgroup_id;
+static u64 wq_low_cgroup_id;
+
+/*
+ * Statistics exposed to userspace through .bss, so the test can verify
+ * that reclaim actually happened instead of relying on timing alone.
+ */
+u64 reclaim_calls;
+u64 reclaimed_bytes;
+
+/*
+ * A timer that fails to rearm stops the whole chain, which from userspace
+ * looks exactly like "the kernel never reclaimed". Count it so the test can
+ * tell the two apart instead of blaming the kernel.
+ */
+u64 timer_failures;
+
+/*
+ * bpf_proactive_reclaim() reports failure as a negative return, which is
+ * otherwise indistinguishable from "this cgroup has nothing left to
+ * reclaim". Keep the last one so a failing kfunc is not mistaken for an
+ * idle cgroup. Stored as a positive errno.
+ */
+u64 last_reclaim_err;
+
+/*
+ * Outcome of a single reclaim attempt, reported to userspace through
+ * the reclaim_events ringbuf so tests can wait for reclaim events with
+ * a timeout instead of polling the counters above.
+ */
+enum reclaim_outcome {
+ RECLAIM_OUTCOME_CALLED,
+ RECLAIM_OUTCOME_SKIPPED_DYING,
+ RECLAIM_OUTCOME_TARGET_GONE,
+};
+
+struct reclaim_event {
+ __u64 outcome;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_RINGBUF);
+ __uint(max_entries, 1 << 16);
+} reclaim_events SEC(".maps");
+
+/*
+ * Reports are best effort: when the ringbuf is full, for example when
+ * userspace does not consume the events, the report is dropped.
+ */
+static void emit_reclaim_event(enum reclaim_outcome outcome)
+{
+ struct reclaim_event *ev;
+
+ ev = bpf_ringbuf_reserve(&reclaim_events, sizeof(*ev), 0);
+ if (!ev)
+ return;
+
+ ev->outcome = outcome;
+ bpf_ringbuf_submit(ev, 0);
+}
+
+static int get_cgroup_memcg_from_id(u64 cgroup_id, struct cgroup_memcg *cm)
+{
+ cm->cgrp = bpf_cgroup_from_id(cgroup_id);
+ if (!cm->cgrp)
+ return -1;
+
+ cm->memcg = bpf_get_mem_cgroup(&cm->cgrp->self);
+ if (!cm->memcg) {
+ bpf_cgroup_release(cm->cgrp);
+ return -1;
+ }
+
+ return 0;
+}
+
+static void put_cgroup_memcg(struct cgroup_memcg *cm)
+{
+ bpf_put_mem_cgroup(cm->memcg);
+ bpf_cgroup_release(cm->cgrp);
+}
+
+static int get_cgroup_event(u64 cgroup_id, u64 *val)
+{
+ struct cgroup_memcg cm;
+
+ if (get_cgroup_memcg_from_id(cgroup_id, &cm))
+ return -1;
+ bpf_mem_cgroup_flush_stats(cm.memcg);
+ *val = bpf_mem_cgroup_page_state(cm.memcg,
+ bpf_core_enum_value(enum node_stat_item,
+ WORKINGSET_REFAULT_FILE));
+ put_cgroup_memcg(&cm);
+
+ return 0;
+}
+
+static bool
+should_reclaim_cgroup(u64 cgroup_id, u64 *prev_event, u64 event_delta_threshold)
+{
+ u64 cur, delta;
+
+ if (get_cgroup_event(cgroup_id, &cur))
+ return false;
+
+ delta = cur - *prev_event;
+ *prev_event = cur;
+
+ return delta >= event_delta_threshold;
+}
+
+/*
+ * rmdir clears CSS_ONLINE on cgrp->self in cgroup_destroy_locked() and only
+ * then deactivates the cgroup's kernfs node with kernfs_remove().
+ * bpf_cgroup_from_id() resolves an id through
+ * kernfs_find_and_get_node_by_id(), which refuses deactivated nodes, so the
+ * lookup starts failing inside the rmdir syscall -- not, as one might expect,
+ * once the cgroup's last reference has been dropped.
+ *
+ * The window in which the lookup still succeeds while the cgroup is already
+ * dying is therefore just the few statements between those two. An attempt
+ * that lands in it reports RECLAIM_OUTCOME_SKIPPED_DYING; after rmdir has
+ * returned, every attempt reports TARGET_GONE instead. The check is kept
+ * because it is the only thing standing between reclaim and a cgroup that is
+ * mid-teardown, but it cannot be hit reliably from userspace, so the test
+ * does not require SKIPPED_DYING.
+ *
+ * Only CSS_ONLINE is worth testing. CSS_DYING is raised by kill_css_sync(),
+ * which dereferences css->ss and so is never called with cgrp->self; the
+ * cgroup's own css only ever loses CSS_ONLINE. This is therefore
+ * cgroup_is_dead() from include/linux/cgroup.h, not css_is_dying().
+ *
+ * CSS_ONLINE comes from vmlinux.h: the kernel defines it in an anonymous
+ * enum, so bpf_core_enum_value() has no enum type to bind to, and
+ * redeclaring it locally would clash with the vmlinux.h enumerators.
+ * vmlinux.h is generated from the running kernel's BTF, so the value
+ * already matches the target kernel.
+ */
+static bool cgroup_is_dying(struct cgroup *cgrp)
+{
+ return !(cgrp->self.flags & CSS_ONLINE);
+}
+
+/*
+ * Reclaim one batch from the target cgroup. Returns the number of
+ * bytes reclaimed, or 0 if the cgroup is dying or gone, the kfunc
+ * failed, or nothing was reclaimed. Each attempt reports its outcome
+ * through the reclaim_events ringbuf, and a failing kfunc additionally
+ * records its errno in last_reclaim_err.
+ */
+static u64 reclaim_cgroup(u64 cgroup_id, u64 size)
+{
+ struct cgroup_memcg cm;
+ long nr;
+
+ if (get_cgroup_memcg_from_id(cgroup_id, &cm)) {
+ emit_reclaim_event(RECLAIM_OUTCOME_TARGET_GONE);
+ return 0;
+ }
+
+ if (cgroup_is_dying(cm.cgrp)) {
+ emit_reclaim_event(RECLAIM_OUTCOME_SKIPPED_DYING);
+ put_cgroup_memcg(&cm);
+ return 0;
+ }
+
+ reclaim_calls++;
+ nr = bpf_proactive_reclaim(cm.memcg, size, -1);
+ if (nr < 0)
+ last_reclaim_err = -nr;
+ else if (nr > 0)
+ reclaimed_bytes += nr;
+ emit_reclaim_event(RECLAIM_OUTCOME_CALLED);
+
+ put_cgroup_memcg(&cm);
+
+ return nr > 0 ? nr : 0;
+}
+
+struct wq_elem {
+ struct bpf_timer timer;
+ struct bpf_wq work;
+ u64 prev_event;
+ u64 event_delta_threshold;
+ u64 check_ns;
+ /*
+ * Bytes still to reclaim in the current round, carried across
+ * requeues. 0 means no round is in progress; the timer path
+ * starts a new round by resetting it, requeued work only looks
+ * at it.
+ */
+ u64 remaining;
+};
+
+struct {
+ __uint(type, BPF_MAP_TYPE_ARRAY);
+ __uint(max_entries, 1);
+ __type(key, __u32);
+ __type(value, struct wq_elem);
+} wq_map SEC(".maps");
+
+static int reclaim_work_fn(void *map, int *key, void *value)
+{
+ struct wq_elem *elem = value;
+ u64 nr, size;
+
+ if (!elem->remaining) {
+ /*
+ * Timer-triggered entry: start a new round only when the
+ * high cgroup refaults enough. Requeued entries skip this
+ * check and only look at remaining, so a round consumes the
+ * refault delta only on the tick that starts it; ticks that
+ * find no round in progress consume and discard it.
+ */
+ if (!should_reclaim_cgroup(wq_high_cgroup_id, &elem->prev_event,
+ elem->event_delta_threshold))
+ return 0;
+ elem->remaining = RECLAIM_MAX_ITER * RECLAIM_SIZE;
+ }
+
+ /* One bounded reclaim pass per callback */
+ size = elem->remaining < RECLAIM_SIZE ? elem->remaining : RECLAIM_SIZE;
+ nr = reclaim_cgroup(wq_low_cgroup_id, size);
+ if (!nr) {
+ elem->remaining = 0;
+ return 0;
+ }
+
+ /* try_to_free_mem_cgroup_pages() may reclaim more than requested */
+ if (nr >= elem->remaining)
+ elem->remaining = 0;
+ else
+ elem->remaining -= nr;
+
+ /* Requeue the same work item for the next batch */
+ if (elem->remaining)
+ bpf_wq_start(&elem->work, 0);
+
+ return 0;
+}
+
+static int wq_timer_cb(void *map, int *key, struct wq_elem *elem)
+{
+ /*
+ * A failed bpf_wq_start() self-heals on the next tick; a failed
+ * rearm does not, so only the latter is worth reporting.
+ */
+ bpf_wq_start(&elem->work, 0);
+ if (bpf_timer_start(&elem->timer, elem->check_ns, 0))
+ timer_failures++;
+
+ return 0;
+}
+
+SEC("syscall")
+int wq_prog_init(struct bpf_args *ctx)
+{
+ struct wq_elem *elem;
+ __u32 key = 0;
+ int ret;
+
+ elem = bpf_map_lookup_elem(&wq_map, &key);
+ if (!elem)
+ return -1;
+
+ ret = bpf_wq_init(&elem->work, &wq_map, 0);
+ if (ret)
+ return ret;
+
+ ret = bpf_wq_set_callback(&elem->work, reclaim_work_fn, 0);
+ if (ret)
+ return ret;
+
+ ret = bpf_timer_init(&elem->timer, &wq_map, CLOCK_MONOTONIC);
+ if (ret)
+ return ret;
+
+ ret = bpf_timer_set_callback(&elem->timer, wq_timer_cb);
+ if (ret)
+ return ret;
+
+ elem->prev_event = 0;
+ elem->remaining = 0;
+ elem->event_delta_threshold = ctx->event_delta_threshold;
+ elem->check_ns = ctx->check_ns;
+
+ wq_high_cgroup_id = ctx->high_cgroup_id;
+ wq_low_cgroup_id = ctx->low_cgroup_id;
+
+ return bpf_timer_start(&elem->timer, elem->check_ns, 0);
+}
+
+char _license[] SEC("license") = "GPL";
--
2.43.0
^ permalink raw reply [flat|nested] 8+ messages in thread