[PATCH] mm/zsmalloc: inline obj_malloc() and obj_free()

[PATCH] mm/zsmalloc: inline obj_malloc() and obj_free()

[Jenhao Chen]

In zs_malloc(), the fast path (if (likely(zspage))) allocates from an
existing zspage via obj_malloc(), while the slow path allocates a new
zspage via alloc_zspage().

Because obj_malloc() has three call sites across zs_malloc() and
migrate_zspage(), Clang at -O2 leaves obj_malloc() out-of-line, adding a
function call on every fast-path allocation. Conversely, because the
362-instruction slow-path alloc_zspage() is static with a single caller,
single-caller inlining under CONFIG_AUTOFDO_CLANG (e.g. with
CONFIG_ZSMALLOC=m) inlines alloc_zspage() into zs_malloc(), growing it
from 254 to 691 instructions.

Mark obj_malloc() and obj_free() __always_inline and mark alloc_zspage()
noinline to keep the fast path inline and the slow-path page allocation
out-of-line.

Signed-off-by: Jenhao Chen <jenhaochen@google.com>
---
 mm/zsmalloc.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 6258a4f5e9fce..5d3443f14a48f 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -960,9 +960,9 @@ static void create_page_chain(struct size_class *class, struct zspage *zspage,
 /*
  * Allocate a zspage for the given size class
  */
-static struct zspage *alloc_zspage(struct zs_pool *pool,
-				   struct size_class *class,
-				   gfp_t gfp, const int nid)
+static noinline struct zspage *alloc_zspage(struct zs_pool *pool,
+					    struct size_class *class,
+					    gfp_t gfp, const int nid)
 {
 	int i;
 	struct zpdesc *zpdescs[ZS_MAX_PAGES_PER_ZSPAGE];
@@ -1209,8 +1209,9 @@ size_t zs_huge_class_size(struct zs_pool *pool)
 }
 EXPORT_SYMBOL_GPL(zs_huge_class_size);
 
-static unsigned long obj_malloc(struct zs_pool *pool,
-				struct zspage *zspage, unsigned long handle)
+static __always_inline unsigned long obj_malloc(struct zs_pool *pool,
+						struct zspage *zspage,
+						unsigned long handle)
 {
 	int i, nr_zpdesc, offset;
 	unsigned long obj;
@@ -1321,7 +1322,7 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp,
 }
 EXPORT_SYMBOL_GPL(zs_malloc);
 
-static void obj_free(int class_size, unsigned long obj)
+static __always_inline void obj_free(int class_size, unsigned long obj)
 {
 	struct link_free *link;
 	struct zspage *zspage;
-- 
2.56.0.317.g454f8302e1-goog