[RFC PATCH 1/3] io_uring/rsrc: allocate io_rsrc_node from a dedicated kmem_cache

[Uzair Beg <uzairbeg11@gmail.com>]

io_rsrc_node allocations come from the generic kmalloc-32 bucket via
io_cache_alloc_new(). On a first fill of a sparse fixed file table the
per-ring node cache is empty by construction, since io_reset_rsrc_node()
returns early on a NULL slot and nothing is freed back, so every install
takes an allocator round trip.

Add an optional kmem_cache to io_alloc_cache and use it for the node
cache. The slab pointer defaults to NULL, so other io_alloc_cache users
are unchanged and imu_cache keeps using kmalloc, its element size being
variable. All three free paths honour the slab.

On its own this is neutral on the reported workload. It exists so that
the following patches can use kmem_cache_alloc_bulk(), which has no
equivalent for plain kmalloc.

Reported-by: Chengfeng Lin <lin2530632123@gmail.com>
Closes: https://lore.kernel.org/io-uring/CANGjgdmt0FQ=offsdfn+wEaDxbOFoAa6bi92X_vEo4S6aCZ56A@mail.gmail.com/
Tested-by: Chengfeng Lin <lin2530632123@gmail.com>
Signed-off-by: Uzair Beg <uzairbeg11@gmail.com>
---
 include/linux/io_uring_types.h |  1 +
 io_uring/alloc_cache.c         | 14 +++++++++++---
 io_uring/alloc_cache.h         |  8 ++++++--
 io_uring/io_uring.c            |  5 +++++
 io_uring/io_uring.h            |  1 +
 io_uring/rsrc.c                |  2 ++
 6 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index c2ea6280901..e8d5a585a60 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -253,6 +253,7 @@ struct io_alloc_cache {
 	unsigned int		max_cached;
 	unsigned int		elem_size;
 	unsigned int		init_clear;
+	struct kmem_cache	*slab;
 };
 
 struct io_ring_ctx {
diff --git a/io_uring/alloc_cache.c b/io_uring/alloc_cache.c
index 58423888b73..a44b82a80f1 100644
--- a/io_uring/alloc_cache.c
+++ b/io_uring/alloc_cache.c
@@ -10,8 +10,12 @@ void io_alloc_cache_free(struct io_alloc_cache *cache,
 	if (!cache->entries)
 		return;
 
-	while ((entry = io_alloc_cache_get(cache)) != NULL)
-		free(entry);
+	while ((entry = io_alloc_cache_get(cache)) != NULL) {
+		if (cache->slab)
+			kmem_cache_free(cache->slab, entry);
+		else
+			free(entry);
+	}
 
 	kvfree(cache->entries);
 	cache->entries = NULL;
@@ -30,6 +34,7 @@ bool io_alloc_cache_init(struct io_alloc_cache *cache,
 	cache->max_cached = max_nr;
 	cache->elem_size = size;
 	cache->init_clear = init_bytes;
+	cache->slab = NULL;
 	return false;
 }
 
@@ -37,7 +42,10 @@ void *io_cache_alloc_new(struct io_alloc_cache *cache, gfp_t gfp)
 {
 	void *obj;
 
-	obj = kmalloc(cache->elem_size, gfp);
+	if (cache->slab)
+		obj = kmem_cache_alloc(cache->slab, gfp);
+	else
+		obj = kmalloc(cache->elem_size, gfp);
 	if (obj && cache->init_clear)
 		memset(obj, 0, cache->init_clear);
 	return obj;
diff --git a/io_uring/alloc_cache.h b/io_uring/alloc_cache.h
index d33ce159ef3..b288bfccc91 100644
--- a/io_uring/alloc_cache.h
+++ b/io_uring/alloc_cache.h
@@ -61,8 +61,12 @@ static inline void *io_cache_alloc(struct io_alloc_cache *cache, gfp_t gfp)
 
 static inline void io_cache_free(struct io_alloc_cache *cache, void *obj)
 {
-	if (!io_alloc_cache_put(cache, obj))
-		kfree(obj);
+	if (!io_alloc_cache_put(cache, obj)) {
+		if (cache->slab)
+			kmem_cache_free(cache->slab, obj);
+		else
+			kfree(obj);
+	}
 }
 
 #endif
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 296667ba712..f375f0ccc0e 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -151,6 +151,7 @@ static void __io_req_caches_free(struct io_ring_ctx *ctx);
 static __read_mostly DEFINE_STATIC_KEY_FALSE(io_key_has_sqarray);
 
 struct kmem_cache *req_cachep;
+struct kmem_cache *io_rsrc_node_cachep;
 static struct workqueue_struct *iou_wq __ro_after_init;
 
 static int __read_mostly sysctl_io_uring_disabled;
@@ -4074,6 +4075,10 @@ static int __init io_uring_init(void)
 				SLAB_HWCACHE_ALIGN | SLAB_PANIC | SLAB_ACCOUNT |
 				SLAB_TYPESAFE_BY_RCU);
 
+	io_rsrc_node_cachep = kmem_cache_create("io_rsrc_node",
+						sizeof(struct io_rsrc_node), NULL,
+						SLAB_HWCACHE_ALIGN | SLAB_PANIC);
+
 	iou_wq = alloc_workqueue("iou_exit", WQ_UNBOUND, 64);
 	BUG_ON(!iou_wq);
 
diff --git a/io_uring/io_uring.h b/io_uring/io_uring.h
index 46d9141d772..6243dadd507 100644
--- a/io_uring/io_uring.h
+++ b/io_uring/io_uring.h
@@ -527,6 +527,7 @@ static inline bool io_req_cache_empty(struct io_ring_ctx *ctx)
 }
 
 extern struct kmem_cache *req_cachep;
+extern struct kmem_cache *io_rsrc_node_cachep;
 
 static inline struct io_kiocb *io_extract_req(struct io_ring_ctx *ctx)
 {
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index d787c16dc1c..6413682ebe4 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -175,6 +175,8 @@ bool io_rsrc_cache_init(struct io_ring_ctx *ctx)
 				  node_size, 0);
 	ret |= io_alloc_cache_init(&ctx->imu_cache, IO_ALLOC_CACHE_MAX,
 				   imu_cache_size, 0);
+	if (!ret)
+		ctx->node_cache.slab = io_rsrc_node_cachep;
 	return ret;
 }
 
-- 
2.43.0