Re: [PATCH] Allocate module.ref array dynamically

[Takashi Iwai <tiwai@suse.de>]

At Tue, 11 Nov 2008 23:06:26 +0100,
Eric Dumazet wrote:
> 
> Takashi Iwai a écrit :
> > Hi,
> > 
> > we found that the kernel module sizes and memory footprints
> > grow drastically when NR_CPUS is high.  For example, with
> > NR_CPUS=4096, SUSE kernel packages weigh over 500MB (even w/o debug
> > info).
> > 
> > A part of the reason is the fixed size array in struct module.
> > The patch below fixes the problem by allocating it dynamically.
> > With the patch, the size can go down to 20MB.
> > 
> > 
> > Any comments/suggestions appreciated.
> > 
> 
> Many attempts were done on this area on the past.
> 
> Your patch has the drawback of using kcalloc(), while previously, module_ref
> space was allocated with vmalloc().
> 
> After a while, a machine could have a lot of vmalloc() space available, but not enough
> physically contiguous space to fullfill a kmalloc(large_area) call.
> 
> So a module load could fail, while previous code could load module.

Then how about to call simply vmalloc() as a fallback?
The revised patch is below.

> I believe Mike Travis has a better patch for this problem, partly using new percpu allocator.

It's interesting...


thanks,

Takashi

===

From: Takashi Iwai <tiwai@suse.de>
Subject: [PATCH] Allocate module.ref array dynamically (v2)

This patch makes the module handling code to allocate the ref
array of each module struct dynamically.  It saves both module
disk space and memory footprints when number of CPUs is high
like 4096.

Reference: Novell bnc#425240
        https://bugzilla.novell.com/show_bug.cgi?id=425240

v1->v2:
 - use vmalloc() as a fallback in case kmalloc() fails

Signed-off-by: Takashi Iwai <tiwai@suse.de>
---
 include/linux/module.h |    2 +-
 kernel/module.c        |   28 +++++++++++++++++++++++-----
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/include/linux/module.h b/include/linux/module.h
index 3bfed01..7b5cbf3 100644
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -348,7 +348,7 @@ struct module
 	void (*exit)(void);
 
 	/* Reference counts */
-	struct module_ref ref[NR_CPUS];
+	struct module_ref *ref;
 #endif
 };
 #ifndef MODULE_ARCH_INIT
diff --git a/kernel/module.c b/kernel/module.c
index 1f4cc00..6fb7ab5 100644
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -571,17 +571,28 @@ static char last_unloaded_module[MODULE_NAME_LEN+1];
 
 #ifdef CONFIG_MODULE_UNLOAD
 /* Init the unload section of the module. */
-static void module_unload_init(struct module *mod)
+static int module_unload_init(struct module *mod)
 {
 	unsigned int i;
+	size_t refsize = nr_cpu_ids * sizeof(*mod->ref);
 
 	INIT_LIST_HEAD(&mod->modules_which_use_me);
-	for (i = 0; i < NR_CPUS; i++)
+
+	mod->ref = kzalloc(refsize, GFP_KERNEL);
+	if (!mod->ref) {
+		mod->ref = vmalloc(refsize);
+		if (!mod->ref)
+			return -ENOMEM;
+		memset(mod->ref, 0, refsize);
+	}
+	for (i = 0; i < nr_cpu_ids; i++)
 		local_set(&mod->ref[i].count, 0);
 	/* Hold reference count during initialization. */
 	local_set(&mod->ref[raw_smp_processor_id()].count, 1);
 	/* Backwards compatibility macros put refcount during init. */
 	mod->waiter = current;
+
+	return 0;
 }
 
 /* modules using other modules */
@@ -661,6 +672,10 @@ static void module_unload_free(struct module *mod)
 			}
 		}
 	}
+	if (is_vmalloc_addr(mod->ref))
+		vfree(mod->ref);
+	else
+		kfree(mod->ref);
 }
 
 #ifdef CONFIG_MODULE_FORCE_UNLOAD
@@ -719,7 +734,7 @@ unsigned int module_refcount(struct module *mod)
 {
 	unsigned int i, total = 0;
 
-	for (i = 0; i < NR_CPUS; i++)
+	for (i = 0; i < nr_cpu_ids; i++)
 		total += local_read(&mod->ref[i].count);
 	return total;
 }
@@ -908,8 +923,9 @@ static inline int use_module(struct module *a, struct module *b)
 	return strong_try_module_get(b) == 0;
 }
 
-static inline void module_unload_init(struct module *mod)
+static inline int module_unload_init(struct module *mod)
 {
+	return 0;
 }
 #endif /* CONFIG_MODULE_UNLOAD */
 
@@ -2051,7 +2067,9 @@ static noinline struct module *load_module(void __user *umod,
 	mod = (void *)sechdrs[modindex].sh_addr;
 
 	/* Now we've moved module, initialize linked lists, etc. */
-	module_unload_init(mod);
+	err = module_unload_init(mod);
+	if (err)
+		goto free_unload;
 
 	/* add kobject, so we can reference it. */
 	err = mod_sysfs_init(mod);
-- 
1.6.0.4