[PATCH] memory hotplug locking: node_size_lock

pgdat->node_size_lock is basically only neeeded in one place in the normal
code: show_mem(), which is the arch-specific sysrq-m printing function.

Strictly speaking, the architectures not doing memory hotplug do no need this
locking in show_mem().  However, they are all included for completeness.  This
should also make any future consolidation of all of the implementations a
little more straightforward.

This lock is also held in the sparsemem code during a memory removal, as
sections are invalidated.  This is the place there pfn_valid() is made false
for a memory area that's being removed.  The lock is only required when doing
pfn_valid() operations on memory which the user does not already have a
reference on the page, such as in show_mem().

Signed-off-by: Dave Hansen <haveblue@us.ibm.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/arch/alpha/mm/numa.c b/arch/alpha/mm/numa.c
index c7481d59b6dff4dc922acb6c6afce238f7a8b939..6d5251254f68f7dd7c8b3797211536b7d40dfff5 100644 (file)
--- a/arch/alpha/mm/numa.c
+++ b/arch/alpha/mm/numa.c
@@ -371,6 +371,8 @@ show_mem(void)
        show_free_areas();
        printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
        for_each_online_node(nid) {
+               unsigned long flags;
+               pgdat_resize_lock(NODE_DATA(nid), &flags);
                i = node_spanned_pages(nid);
                while (i-- > 0) {
                        struct page *page = nid_page_nr(nid, i);
@@ -384,6 +386,7 @@ show_mem(void)
                        else
                                shared += page_count(page) - 1;
                }
+               pgdat_resize_unlock(NODE_DATA(nid), &flags);
        }
        printk("%ld pages of RAM\n",total);
        printk("%ld free pages\n",free);

diff --git a/arch/i386/mm/pgtable.c b/arch/i386/mm/pgtable.c
index 39c099f15b5f84c0eb00010b8010c46da421c8af..9db3242103bea9da0b03240fa129b59f1c45ea81 100644 (file)
--- a/arch/i386/mm/pgtable.c
+++ b/arch/i386/mm/pgtable.c
@@ -31,11 +31,13 @@ void show_mem(void)
        pg_data_t *pgdat;
        unsigned long i;
        struct page_state ps;
+       unsigned long flags;
 
        printk(KERN_INFO "Mem-info:\n");
        show_free_areas();
        printk(KERN_INFO "Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
        for_each_pgdat(pgdat) {
+               pgdat_resize_lock(pgdat, &flags);
                for (i = 0; i < pgdat->node_spanned_pages; ++i) {
                        page = pgdat_page_nr(pgdat, i);
                        total++;
@@ -48,6 +50,7 @@ void show_mem(void)
                        else if (page_count(page))
                                shared += page_count(page) - 1;
                }
+               pgdat_resize_unlock(pgdat, &flags);
        }
        printk(KERN_INFO "%d pages of RAM\n", total);
        printk(KERN_INFO "%d pages of HIGHMEM\n", highmem);

diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index a3788fb84809a0c1c1f565f853d639bb219290e0..a88cdb7232f8ea1586473973c65f9a63d8ecdde7 100644 (file)
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -555,9 +555,13 @@ void show_mem(void)
        show_free_areas();
        printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
        for_each_pgdat(pgdat) {
-               unsigned long present = pgdat->node_present_pages;
+               unsigned long present;
+               unsigned long flags;
                int shared = 0, cached = 0, reserved = 0;
+
                printk("Node ID: %d\n", pgdat->node_id);
+               pgdat_resize_lock(pgdat, &flags);
+               present = pgdat->node_present_pages;
                for(i = 0; i < pgdat->node_spanned_pages; i++) {
                        struct page *page;
                        if (pfn_valid(pgdat->node_start_pfn + i))
@@ -571,6 +575,7 @@ void show_mem(void)
                        else if (page_count(page))
                                shared += page_count(page)-1;
                }
+               pgdat_resize_unlock(pgdat, &flags);
                total_present += present;
                total_reserved += reserved;
                total_cached += cached;

diff --git a/arch/m32r/mm/init.c b/arch/m32r/mm/init.c
index d9a40b1fe8ba84e8a8ad4d36920f9a0ecba1c357..6facf15b04f30d1353840e65fa8362fc664c4be1 100644 (file)
--- a/arch/m32r/mm/init.c
+++ b/arch/m32r/mm/init.c
@@ -48,6 +48,8 @@ void show_mem(void)
        show_free_areas();
        printk("Free swap:       %6ldkB\n",nr_swap_pages<<(PAGE_SHIFT-10));
        for_each_pgdat(pgdat) {
+               unsigned long flags;
+               pgdat_resize_lock(pgdat, &flags);
                for (i = 0; i < pgdat->node_spanned_pages; ++i) {
                        page = pgdat_page_nr(pgdat, i);
                        total++;
@@ -60,6 +62,7 @@ void show_mem(void)
                        else if (page_count(page))
                                shared += page_count(page) - 1;
                }
+               pgdat_resize_unlock(pgdat, &flags);
        }
        printk("%d pages of RAM\n", total);
        printk("%d pages of HIGHMEM\n",highmem);
@@ -150,10 +153,14 @@ int __init reservedpages_count(void)
        int reservedpages, nid, i;
 
        reservedpages = 0;
-       for_each_online_node(nid)
+       for_each_online_node(nid) {
+               unsigned long flags;
+               pgdat_resize_lock(NODE_DATA(nid), &flags);
                for (i = 0 ; i < MAX_LOW_PFN(nid) - START_PFN(nid) ; i++)
                        if (PageReserved(nid_page_nr(nid, i)))
                                reservedpages++;
+               pgdat_resize_unlock(NODE_DATA(nid), &flags);
+       }
 
        return reservedpages;
 }

diff --git a/arch/parisc/mm/init.c b/arch/parisc/mm/init.c
index 2886ad70db48a5d44eb16fb8aa9aa4262c701d4f..29b998e430e6023c4c823b545b1f3e8b5e700a46 100644 (file)
--- a/arch/parisc/mm/init.c
+++ b/arch/parisc/mm/init.c
@@ -505,7 +505,9 @@ void show_mem(void)
 
                for (j = node_start_pfn(i); j < node_end_pfn(i); j++) {
                        struct page *p;
+                       unsigned long flags;
 
+                       pgdat_resize_lock(NODE_DATA(i), &flags);
                        p = nid_page_nr(i, j) - node_start_pfn(i);
 
                        total++;
@@ -517,6 +519,7 @@ void show_mem(void)
                                free++;
                        else
                                shared += page_count(p) - 1;
+                       pgdat_resize_unlock(NODE_DATA(i), &flags);
                }
        }
 #endif

diff --git a/arch/ppc64/mm/init.c b/arch/ppc64/mm/init.c
index a45584b3440c850ab288a16c69fbe893afe0f334..975b26de34d6dc56f20f42403717291b87a68f6c 100644 (file)
--- a/arch/ppc64/mm/init.c
+++ b/arch/ppc64/mm/init.c
@@ -104,6 +104,8 @@ void show_mem(void)
        show_free_areas();
        printk("Free swap:       %6ldkB\n", nr_swap_pages<<(PAGE_SHIFT-10));
        for_each_pgdat(pgdat) {
+               unsigned long flags;
+               pgdat_resize_lock(pgdat, &flags);
                for (i = 0; i < pgdat->node_spanned_pages; i++) {
                        page = pgdat_page_nr(pgdat, i);
                        total++;
@@ -114,6 +116,7 @@ void show_mem(void)
                        else if (page_count(page))
                                shared += page_count(page) - 1;
                }
+               pgdat_resize_unlock(pgdat, &flags);
        }
        printk("%ld pages of RAM\n", total);
        printk("%ld reserved pages\n", reserved);
@@ -647,11 +650,14 @@ void __init mem_init(void)
 #endif
 
        for_each_pgdat(pgdat) {
+               unsigned long flags;
+               pgdat_resize_lock(pgdat, &flags);
                for (i = 0; i < pgdat->node_spanned_pages; i++) {
                        page = pgdat_page_nr(pgdat, i);
                        if (PageReserved(page))
                                reservedpages++;
                }
+               pgdat_resize_unlock(pgdat, &flags);
        }
 
        codesize = (unsigned long)&_etext - (unsigned long)&_stext;

diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
new file mode 100644 (file)
index 0000000..e8103be
--- /dev/null
+++ b/include/linux/memory_hotplug.h
@@ -0,0 +1,34 @@
+#ifndef __LINUX_MEMORY_HOTPLUG_H
+#define __LINUX_MEMORY_HOTPLUG_H
+
+#include <linux/mmzone.h>
+#include <linux/spinlock.h>
+
+#ifdef CONFIG_MEMORY_HOTPLUG
+/*
+ * pgdat resizing functions
+ */
+static inline
+void pgdat_resize_lock(struct pglist_data *pgdat, unsigned long *flags)
+{
+       spin_lock_irqsave(&pgdat->node_size_lock, *flags);
+}
+static inline
+void pgdat_resize_unlock(struct pglist_data *pgdat, unsigned long *flags)
+{
+       spin_lock_irqrestore(&pgdat->node_size_lock, *flags);
+}
+static inline
+void pgdat_resize_init(struct pglist_data *pgdat)
+{
+       spin_lock_init(&pgdat->node_size_lock);
+}
+#else /* ! CONFIG_MEMORY_HOTPLUG */
+/*
+ * Stub functions for when hotplug is off
+ */
+static inline void pgdat_resize_lock(struct pglist_data *p, unsigned long *f) {}
+static inline void pgdat_resize_unlock(struct pglist_data *p, unsigned long *f) {}
+static inline void pgdat_resize_init(struct pglist_data *pgdat) {}
+#endif
+#endif /* __LINUX_MEMORY_HOTPLUG_H */

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 4674145bb63d650d6f376c94dd3f0199815820c9..e050d68963a126ec06dbe7cfdee2b59722c3071f 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -273,6 +273,16 @@ typedef struct pglist_data {
        struct page *node_mem_map;
 #endif
        struct bootmem_data *bdata;
+#ifdef CONFIG_MEMORY_HOTPLUG
+       /*
+        * Must be held any time you expect node_start_pfn, node_present_pages
+        * or node_spanned_pages stay constant.  Holding this will also
+        * guarantee that any pfn_valid() stays that way.
+        *
+        * Nests above zone->lock and zone->size_seqlock.
+        */
+       spinlock_t node_size_lock;
+#endif
        unsigned long node_start_pfn;
        unsigned long node_present_pages; /* total number of physical pages */
        unsigned long node_spanned_pages; /* total size of physical page
@@ -293,6 +303,8 @@ typedef struct pglist_data {
 #endif
 #define nid_page_nr(nid, pagenr)       pgdat_page_nr(NODE_DATA(nid),(pagenr))
 
+#include <linux/memory_hotplug.h>
+
 extern struct pglist_data *pgdat_list;
 
 void __get_zone_counts(unsigned long *active, unsigned long *inactive,

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index a51ef94eec33970e41f64d4b327d7e4686f70f15..32fad6d23200f7333f4635b128b78a4cefe37c79 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1958,6 +1958,7 @@ static void __init free_area_init_core(struct pglist_data *pgdat,
        int nid = pgdat->node_id;
        unsigned long zone_start_pfn = pgdat->node_start_pfn;
 
+       pgdat_resize_init(pgdat);
        pgdat->nr_zones = 0;
        init_waitqueue_head(&pgdat->kswapd_wait);
        pgdat->kswapd_max_order = 0;