vmscan: move isolate_lru_page() to vmscan.c

On large memory systems, the VM can spend way too much time scanning
through pages that it cannot (or should not) evict from memory.  Not only
does it use up CPU time, but it also provokes lock contention and can
leave large systems under memory presure in a catatonic state.

This patch series improves VM scalability by:

1) putting filesystem backed, swap backed and unevictable pages
   onto their own LRUs, so the system only scans the pages that it
   can/should evict from memory

2) switching to two handed clock replacement for the anonymous LRUs,
   so the number of pages that need to be scanned when the system
   starts swapping is bound to a reasonable number

3) keeping unevictable pages off the LRU completely, so the
   VM does not waste CPU time scanning them. ramfs, ramdisk,
   SHM_LOCKED shared memory segments and mlock()ed VMA pages
   are keept on the unevictable list.

This patch:

isolate_lru_page logically belongs to be in vmscan.c than migrate.c.

It is tough, because we don't need that function without memory migration
so there is a valid argument to have it in migrate.c.  However a
subsequent patch needs to make use of it in the core mm, so we can happily
move it to vmscan.c.

Also, make the function a little more generic by not requiring that it
adds an isolated page to a given list.  Callers can do that.

	Note that we now have '__isolate_lru_page()', that does
	something quite different, visible outside of vmscan.c
	for use with memory controller.  Methinks we need to
	rationalize these names/purposes.	--lts

[akpm@linux-foundation.org: fix mm/memory_hotplug.c build]
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Rik van Riel <riel@redhat.com>
Signed-off-by: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 03aea612d28450f24f96274c922a8da987bb918b..3f34005068d420b2b1766afdec804529dc2772d9 100644 (file)
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -7,7 +7,6 @@
 typedef struct page *new_page_t(struct page *, unsigned long private, int **);
 
 #ifdef CONFIG_MIGRATION
-extern int isolate_lru_page(struct page *p, struct list_head *pagelist);
 extern int putback_lru_pages(struct list_head *l);
 extern int migrate_page(struct address_space *,
                        struct page *, struct page *);
@@ -21,8 +20,6 @@ extern int migrate_vmas(struct mm_struct *mm,
                const nodemask_t *from, const nodemask_t *to,
                unsigned long flags);
 #else
-static inline int isolate_lru_page(struct page *p, struct list_head *list)
-                                       { return -ENOSYS; }
 static inline int putback_lru_pages(struct list_head *l) { return 0; }
 static inline int migrate_pages(struct list_head *l, new_page_t x,
                unsigned long private) { return -ENOSYS; }

diff --git a/mm/internal.h b/mm/internal.h
index 1f43f7416972d3c4f1b07ab04898008a8ec76a81..4e8e78b978b5b89031ee4ceb2a6c9fb324824bd7 100644 (file)
--- a/mm/internal.h
+++ b/mm/internal.h
@@ -39,6 +39,8 @@ static inline void __put_page(struct page *page)
        atomic_dec(&page->_count);
 }
 
+extern int isolate_lru_page(struct page *page);
+
 extern void __free_pages_bootmem(struct page *page, unsigned int order);
 
 /*

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index c299d083d8e27634b1a19af91ddb86836c787380..3b4975815141f3f198926176d35c60a6fc282635 100644 (file)
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -658,8 +658,9 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
                 * We can skip free pages. And we can only deal with pages on
                 * LRU.
                 */
-               ret = isolate_lru_page(page, &source);
+               ret = isolate_lru_page(page);
                if (!ret) { /* Success */
+                       list_add_tail(&page->lru, &source);
                        move_pages--;
                } else {
                        /* Becasue we don't have big zone->lock. we should

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 83369058ec133b380a482ca86792087134b944f6..71b47491487d013a04a23901a3c5eefc2fc33389 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -93,6 +93,8 @@
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 
+#include "internal.h"
+
 /* Internal flags */
 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)   /* Skip checks for continuous vmas */
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)         /* Invert check for nodemask */
@@ -762,8 +764,11 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
        /*
         * Avoid migrating a page that is shared with others.
         */
-       if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
-               isolate_lru_page(page, pagelist);
+       if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
+               if (!isolate_lru_page(page)) {
+                       list_add_tail(&page->lru, pagelist);
+               }
+       }
 }
 
 static struct page *new_node_page(struct page *page, unsigned long node, int **x)

diff --git a/mm/migrate.c b/mm/migrate.c
index 2a80136b23bbc16fb9d2ec2fad932b29e30dda50..da73742e52a54d2aca016bc46ca74e152c447a40 100644 (file)
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -36,36 +36,6 @@
 
 #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
 
-/*
- * Isolate one page from the LRU lists. If successful put it onto
- * the indicated list with elevated page count.
- *
- * Result:
- *  -EBUSY: page not on LRU list
- *  0: page removed from LRU list and added to the specified list.
- */
-int isolate_lru_page(struct page *page, struct list_head *pagelist)
-{
-       int ret = -EBUSY;
-
-       if (PageLRU(page)) {
-               struct zone *zone = page_zone(page);
-
-               spin_lock_irq(&zone->lru_lock);
-               if (PageLRU(page) && get_page_unless_zero(page)) {
-                       ret = 0;
-                       ClearPageLRU(page);
-                       if (PageActive(page))
-                               del_page_from_active_list(zone, page);
-                       else
-                               del_page_from_inactive_list(zone, page);
-                       list_add_tail(&page->lru, pagelist);
-               }
-               spin_unlock_irq(&zone->lru_lock);
-       }
-       return ret;
-}
-
 /*
  * migrate_prep() needs to be called before we start compiling a list of pages
  * to be migrated using isolate_lru_page().
@@ -914,7 +884,9 @@ static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
                                !migrate_all)
                        goto put_and_set;
 
-               err = isolate_lru_page(page, &pagelist);
+               err = isolate_lru_page(page);
+               if (!err)
+                       list_add_tail(&page->lru, &pagelist);
 put_and_set:
                /*
                 * Either remove the duplicate refcount from

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 1ff1a58e7c1075fffecda7eb9ce5c360e8369a17..1fd4912a596c347f0eb699c5d25a085f80da6ef9 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -844,6 +844,51 @@ static unsigned long clear_active_flags(struct list_head *page_list)
        return nr_active;
 }
 
+/**
+ * isolate_lru_page - tries to isolate a page from its LRU list
+ * @page: page to isolate from its LRU list
+ *
+ * Isolates a @page from an LRU list, clears PageLRU and adjusts the
+ * vmstat statistic corresponding to whatever LRU list the page was on.
+ *
+ * Returns 0 if the page was removed from an LRU list.
+ * Returns -EBUSY if the page was not on an LRU list.
+ *
+ * The returned page will have PageLRU() cleared.  If it was found on
+ * the active list, it will have PageActive set.  That flag may need
+ * to be cleared by the caller before letting the page go.
+ *
+ * The vmstat statistic corresponding to the list on which the page was
+ * found will be decremented.
+ *
+ * Restrictions:
+ * (1) Must be called with an elevated refcount on the page. This is a
+ *     fundamentnal difference from isolate_lru_pages (which is called
+ *     without a stable reference).
+ * (2) the lru_lock must not be held.
+ * (3) interrupts must be enabled.
+ */
+int isolate_lru_page(struct page *page)
+{
+       int ret = -EBUSY;
+
+       if (PageLRU(page)) {
+               struct zone *zone = page_zone(page);
+
+               spin_lock_irq(&zone->lru_lock);
+               if (PageLRU(page) && get_page_unless_zero(page)) {
+                       ret = 0;
+                       ClearPageLRU(page);
+                       if (PageActive(page))
+                               del_page_from_active_list(zone, page);
+                       else
+                               del_page_from_inactive_list(zone, page);
+               }
+               spin_unlock_irq(&zone->lru_lock);
+       }
+       return ret;
+}
+
 /*
  * shrink_inactive_list() is a helper for shrink_zone().  It returns the number
  * of reclaimed pages