modules: Use a better scheme for refcounting

Current refcounting for modules (done if CONFIG_MODULE_UNLOAD=y) is
using a lot of memory.

Each 'struct module' contains an [NR_CPUS] array of full cache lines.

This patch uses existing infrastructure (percpu_modalloc() &
percpu_modfree()) to allocate percpu space for the refcount storage.

Instead of wasting NR_CPUS*128 bytes (on i386), we now use
nr_cpu_ids*sizeof(local_t) bytes.

On a typical distro, where NR_CPUS=8, shiping 2000 modules, we reduce
size of module files by about 2 Mbytes. (1Kb per module)

Instead of having all refcounters in the same memory node - with TLB misses
because of vmalloc() - this new implementation permits to have better
NUMA properties, since each  CPU will use storage on its preferred node,
thanks to percpu storage.

Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/module.h b/include/linux/module.h
index 4f7ea12463d33d3ec99ad98da98ec8b1cba52a80..f3b8329eb5b877d7233ab66d566dd280fc78536c 100644 (file)
--- a/include/linux/module.h
+++ b/include/linux/module.h
@@ -219,11 +219,6 @@ void *__symbol_get_gpl(const char *symbol);
 
 #endif
 
-struct module_ref
-{
-       local_t count;
-} ____cacheline_aligned;
-
 enum module_state
 {
        MODULE_STATE_LIVE,
@@ -344,8 +339,11 @@ struct module
        /* Destruction function. */
        void (*exit)(void);
 
-       /* Reference counts */
-       struct module_ref ref[NR_CPUS];
+#ifdef CONFIG_SMP
+       char *refptr;
+#else
+       local_t ref;
+#endif
 #endif
 };
 #ifndef MODULE_ARCH_INIT
@@ -395,13 +393,22 @@ void __symbol_put(const char *symbol);
 #define symbol_put(x) __symbol_put(MODULE_SYMBOL_PREFIX #x)
 void symbol_put_addr(void *addr);
 
+static inline local_t *__module_ref_addr(struct module *mod, int cpu)
+{
+#ifdef CONFIG_SMP
+       return (local_t *) (mod->refptr + per_cpu_offset(cpu));
+#else
+       return &mod->ref;
+#endif
+}
+
 /* Sometimes we know we already have a refcount, and it's easier not
    to handle the error case (which only happens with rmmod --wait). */
 static inline void __module_get(struct module *module)
 {
        if (module) {
                BUG_ON(module_refcount(module) == 0);
-               local_inc(&module->ref[get_cpu()].count);
+               local_inc(__module_ref_addr(module, get_cpu()));
                put_cpu();
        }
 }
@@ -413,7 +420,7 @@ static inline int try_module_get(struct module *module)
        if (module) {
                unsigned int cpu = get_cpu();
                if (likely(module_is_live(module)))
-                       local_inc(&module->ref[cpu].count);
+                       local_inc(__module_ref_addr(module, cpu));
                else
                        ret = 0;
                put_cpu();

diff --git a/kernel/module.c b/kernel/module.c
index e8b51d41dd72cf8170d3404b14dff167978e324a..ba22484a987eb96a9d2bff0cdeede4b62a1d7156 100644 (file)
--- a/kernel/module.c
+++ b/kernel/module.c
@@ -573,13 +573,13 @@ static char last_unloaded_module[MODULE_NAME_LEN+1];
 /* Init the unload section of the module. */
 static void module_unload_init(struct module *mod)
 {
-       unsigned int i;
+       int cpu;
 
        INIT_LIST_HEAD(&mod->modules_which_use_me);
-       for (i = 0; i < NR_CPUS; i++)
-               local_set(&mod->ref[i].count, 0);
+       for_each_possible_cpu(cpu)
+               local_set(__module_ref_addr(mod, cpu), 0);
        /* Hold reference count during initialization. */
-       local_set(&mod->ref[raw_smp_processor_id()].count, 1);
+       local_set(__module_ref_addr(mod, raw_smp_processor_id()), 1);
        /* Backwards compatibility macros put refcount during init. */
        mod->waiter = current;
 }
@@ -717,10 +717,11 @@ static int try_stop_module(struct module *mod, int flags, int *forced)
 
 unsigned int module_refcount(struct module *mod)
 {
-       unsigned int i, total = 0;
+       unsigned int total = 0;
+       int cpu;
 
-       for (i = 0; i < NR_CPUS; i++)
-               total += local_read(&mod->ref[i].count);
+       for_each_possible_cpu(cpu)
+               total += local_read(__module_ref_addr(mod, cpu));
        return total;
 }
 EXPORT_SYMBOL(module_refcount);
@@ -894,7 +895,7 @@ void module_put(struct module *module)
 {
        if (module) {
                unsigned int cpu = get_cpu();
-               local_dec(&module->ref[cpu].count);
+               local_dec(__module_ref_addr(module, cpu));
                /* Maybe they're waiting for us to drop reference? */
                if (unlikely(!module_is_live(module)))
                        wake_up_process(module->waiter);
@@ -1464,7 +1465,10 @@ static void free_module(struct module *mod)
        kfree(mod->args);
        if (mod->percpu)
                percpu_modfree(mod->percpu);
-
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+       if (mod->refptr)
+               percpu_modfree(mod->refptr);
+#endif
        /* Free lock-classes: */
        lockdep_free_key_range(mod->module_core, mod->core_size);
 
@@ -2011,6 +2015,14 @@ static noinline struct module *load_module(void __user *umod,
        if (err < 0)
                goto free_mod;
 
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+       mod->refptr = percpu_modalloc(sizeof(local_t), __alignof__(local_t),
+                                     mod->name);
+       if (!mod->refptr) {
+               err = -ENOMEM;
+               goto free_mod;
+       }
+#endif
        if (pcpuindex) {
                /* We have a special allocation for this section. */
                percpu = percpu_modalloc(sechdrs[pcpuindex].sh_size,
@@ -2018,7 +2030,7 @@ static noinline struct module *load_module(void __user *umod,
                                         mod->name);
                if (!percpu) {
                        err = -ENOMEM;
-                       goto free_mod;
+                       goto free_percpu;
                }
                sechdrs[pcpuindex].sh_flags &= ~(unsigned long)SHF_ALLOC;
                mod->percpu = percpu;
@@ -2282,6 +2294,9 @@ static noinline struct module *load_module(void __user *umod,
  free_percpu:
        if (percpu)
                percpu_modfree(percpu);
+#if defined(CONFIG_MODULE_UNLOAD) && defined(CONFIG_SMP)
+       percpu_modfree(mod->refptr);
+#endif
  free_mod:
        kfree(args);
  free_hdr: