[RFC PATCH v25 11/14] mm: shmem: use a limit counter for the tmpfs size limit

[Mathieu Desnoyers <mathieu.desnoyers@efficios.com>]

The size limit of tmpfs (the size= mount option) is a struct
percpu_counter. Each block allocation checks it with
percpu_counter_limited_add(). That check takes no lock only while the
mount is more than batch * nr_cpus pages below its size. With the
default batch size of 768 at 384 CPUs, this is 294912 pages, 1.1 GB.
A smaller mount, such as a 64 MB /dev/shm, takes the counter spinlock
and sums all CPUs on every page. On a 384-CPU host with 384 processes
allocating, 99.5% of the time goes to that spinlock.

Use a hierarchical limit counter instead. Its batch size is chosen from
the size. When the size allows, its accuracy is within 1/4 of the size.
Block allocations then sum the CPUs only in the last 1/4 of the size.
They take no lock. The limit stays strict.

Near a full mount, an allocation can fail although it fits. This
happens while a concurrent allocation of a few pages, which does not
fit, is being undone. A large folio is checked before it is added, so a
large folio which does not fit has no such effect.

A mount without a size limit never counts its blocks. A remount cannot
add a limit. So such a mount no longer allocates the counter.

A remount keeps the batch size. A mount grown by remount thus keeps the
smaller batch size of its first size. Under concurrent allocations, a
remount to a smaller size can be refused although the use fits.

Test: fallocate() of one page at a time into one file per process, all
on one mount. This is a will-it-scale variant. It ran in a 384-vCPU
guest pinned to a 2-socket AMD EPYC 9654 host, on v7.4-rc2. In
thousands of calls per second:

  size=  processes   before    after
  64 MB          1      208      483
  64 MB         64      256    17771
  64 MB        384      126    54275
  1 GB         384      125    93879
  2 GB         384    93851    93476
  16 GB        384    93737    94806

Below 1.1 GB, the mount scales again with the number of processes. At
2 GB and above, the results are within 1% of before. Within the last
1/4 of the size, each allocation sums both counters over all CPUs. With
one process, this is 37% slower than before. With 384 processes, it is
about 100 times faster.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
---
 include/linux/shmem_fs.h |  3 ++-
 mm/shmem.c               | 33 +++++++++++++++++++++++----------
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h
index f89b4b151133..b37adf32e4ef 100644
--- a/include/linux/shmem_fs.h
+++ b/include/linux/shmem_fs.h
@@ -7,6 +7,7 @@
 #include <linux/mempolicy.h>
 #include <linux/pagemap.h>
 #include <linux/percpu_counter.h>
+#include <linux/percpu_counter_tree.h>
 #include <linux/xattr.h>
 #include <linux/fs_parser.h>
 #include <linux/userfaultfd_k.h>
@@ -76,7 +77,7 @@ struct shmem_quota_limits {
 
 struct shmem_sb_info {
 	unsigned long max_blocks;   /* How many blocks are allowed */
-	struct percpu_counter used_blocks;  /* How many are allocated */
+	struct percpu_counter_tree_limit used_blocks; /* How many are allocated */
 	unsigned long max_inodes;   /* How many inodes are allowed */
 	unsigned long free_ispace;  /* How much ispace left for allocation */
 	raw_spinlock_t stat_lock;   /* Serialize shmem_sb_info changes */
diff --git a/mm/shmem.c b/mm/shmem.c
index b8399bb71997..2e368d7b0e41 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -231,13 +231,13 @@ int shmem_inode_acct_blocks(struct inode *inode, long pages)
 
 	might_sleep();	/* when quotas */
 	if (sbinfo->max_blocks) {
-		if (!percpu_counter_limited_add(&sbinfo->used_blocks,
-						sbinfo->max_blocks, pages))
+		if (!percpu_counter_tree_limit_add(&sbinfo->used_blocks,
+						   sbinfo->max_blocks, pages))
 			goto unacct;
 
 		err = dquot_alloc_block_nodirty(inode, pages);
 		if (err) {
-			percpu_counter_sub(&sbinfo->used_blocks, pages);
+			percpu_counter_tree_limit_sub(&sbinfo->used_blocks, pages);
 			goto unacct;
 		}
 	} else {
@@ -262,7 +262,7 @@ static void shmem_inode_unacct_blocks(struct inode *inode, long pages)
 	dquot_free_block_nodirty(inode, pages);
 
 	if (sbinfo->max_blocks)
-		percpu_counter_sub(&sbinfo->used_blocks, pages);
+		percpu_counter_tree_limit_sub(&sbinfo->used_blocks, pages);
 	shmem_unacct_blocks(info->flags, pages);
 }
 
@@ -4048,9 +4048,11 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
 	buf->f_namelen = NAME_MAX;
 	if (sbinfo->max_blocks) {
 		buf->f_blocks = sbinfo->max_blocks;
+		/* Under concurrent allocations, the sum can be above the size. */
 		buf->f_bavail =
 		buf->f_bfree  = sbinfo->max_blocks -
-				percpu_counter_sum(&sbinfo->used_blocks);
+				min(percpu_counter_tree_limit_precise_sum(&sbinfo->used_blocks),
+				    sbinfo->max_blocks);
 	}
 	if (sbinfo->max_inodes) {
 		buf->f_files = sbinfo->max_inodes;
@@ -5020,8 +5022,8 @@ static int shmem_reconfigure(struct fs_context *fc)
 			err = "Cannot retroactively limit size";
 			goto out;
 		}
-		if (percpu_counter_compare(&sbinfo->used_blocks,
-					   ctx->blocks) > 0) {
+		if (percpu_counter_tree_limit_precise_sum(&sbinfo->used_blocks) >
+		    ctx->blocks) {
 			err = "Too small a size for current use";
 			goto out;
 		}
@@ -5187,7 +5189,7 @@ static void shmem_put_super(struct super_block *sb)
 	shmem_disable_quotas(sb);
 #endif
 	free_percpu(sbinfo->ino_batch);
-	percpu_counter_destroy(&sbinfo->used_blocks);
+	percpu_counter_tree_limit_destroy(&sbinfo->used_blocks);
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	list_lru_destroy(&sbinfo->shrinklist);
 #endif
@@ -5212,6 +5214,7 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
 	static struct lock_class_key shmem_unused_huge_lock_key;
 #endif
 	struct shmem_options *ctx = fc->fs_private;
+	unsigned long batch;
 	struct inode *inode;
 	struct shmem_sb_info *sbinfo;
 	int error = -ENOMEM;
@@ -5286,8 +5289,18 @@ static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
 	ctx->mpol = NULL;
 
 	raw_spin_lock_init(&sbinfo->stat_lock);
-	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
-		goto failed;
+	/*
+	 * A mount without a size limit never counts its blocks. A remount
+	 * cannot add a limit. Otherwise, pick the batch size from the size.
+	 * When the size allows, its accuracy is within 1/4 of the size. A
+	 * block allocation then sums all CPUs only within the last 1/4 of the
+	 * size. A remount keeps the batch size.
+	 */
+	if (sbinfo->max_blocks) {
+		batch = percpu_counter_tree_batch_for_accuracy(sbinfo->max_blocks / 4);
+		if (percpu_counter_tree_limit_init(&sbinfo->used_blocks, batch, GFP_KERNEL))
+			goto failed;
+	}
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 	if (list_lru_init_memcg_key(&sbinfo->shrinklist, sb->s_shrink,
-- 
2.47.3