mm: optimize compound_head() by avoiding a shared page flag

The patch adds PageTail(page) and PageHead(page) to check if a page is the
head or the tail of a compound page.  This is done by masking the two bits
describing the state of a compound page and then comparing them.  So one
comparision and a branch instead of two bit checks and two branches.

Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 8c149fa4491d751bd70997b024d220a5a139f8d4..695b90437bbc763bba0bb935793450d102f1dbf7 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -269,14 +269,7 @@ static inline int get_page_unless_zero(struct page *page)
 
 static inline struct page *compound_head(struct page *page)
 {
-       /*
-        * We could avoid the PageCompound(page) check if
-        * we would not overload PageTail().
-        *
-        * This check has to be done in several performance critical
-        * paths of the slab etc. IMHO PageTail deserves its own flag.
-        */
-       if (unlikely(PageCompound(page) && PageTail(page)))
+       if (unlikely(PageTail(page)))
                return page->first_page;
        return page;
 }
@@ -327,7 +320,7 @@ static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
 
 static inline int compound_order(struct page *page)
 {
-       if (!PageCompound(page) || PageTail(page))
+       if (!PageHead(page))
                return 0;
        return (unsigned long)page[1].lru.prev;
 }

diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index a1e143634946f08128737148f53358550b356ec8..a3c8b60a9c3a72519ef2356a5bf4b2ecb9000c35 100644 (file)
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -6,6 +6,7 @@
 #define PAGE_FLAGS_H
 
 #include <linux/types.h>
+#include <linux/mm_types.h>
 
 /*
  * Various page->flags bits:
@@ -94,12 +95,6 @@
 /* PG_owner_priv_1 users should have descriptive aliases */
 #define PG_checked             PG_owner_priv_1 /* Used by some filesystems */
 
-/*
- * Marks tail portion of a compound page. We currently do not reclaim
- * compound pages so we can reuse a flag only used for reclaim here.
- */
-#define PG_tail                        PG_reclaim
-
 #if (BITS_PER_LONG > 32)
 /*
  * 64-bit-only flags build down from bit 31
@@ -248,12 +243,32 @@ static inline void SetPageUptodate(struct page *page)
 #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
 
 /*
- * Note: PG_tail is an alias of another page flag. The result of PageTail()
- * is only valid if PageCompound(page) is true.
+ * PG_reclaim is used in combination with PG_compound to mark the
+ * head and tail of a compound page
+ *
+ * PG_compound & PG_reclaim    => Tail page
+ * PG_compound & ~PG_reclaim   => Head page
  */
-#define PageTail(page) test_bit(PG_tail, &(page)->flags)
-#define __SetPageTail(page)    __set_bit(PG_tail, &(page)->flags)
-#define __ClearPageTail(page)  __clear_bit(PG_tail, &(page)->flags)
+
+#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
+
+#define PageTail(page) ((page->flags & PG_head_tail_mask) \
+                               == PG_head_tail_mask)
+
+static inline void __SetPageTail(struct page *page)
+{
+       page->flags |= PG_head_tail_mask;
+}
+
+static inline void __ClearPageTail(struct page *page)
+{
+       page->flags &= ~PG_head_tail_mask;
+}
+
+#define PageHead(page) ((page->flags & PG_head_tail_mask) \
+                               == (1L << PG_compound))
+#define __SetPageHead(page)    __SetPageCompound(page)
+#define __ClearPageHead(page)  __ClearPageCompound(page)
 
 #ifdef CONFIG_SWAP
 #define PageSwapCache(page)    test_bit(PG_swapcache, &(page)->flags)

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index fc241fe295abf21a9c015b66a21d0e1eb18ea7d8..36d713e216e8d4dd22d42b3c1f72b570ff0965ea 100644 (file)
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -235,12 +235,11 @@ static void prep_compound_page(struct page *page, unsigned long order)
 
        set_compound_page_dtor(page, free_compound_page);
        set_compound_order(page, order);
-       __SetPageCompound(page);
+       __SetPageHead(page);
        for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
                __SetPageTail(p);
-               __SetPageCompound(p);
                p->first_page = page;
        }
 }
@@ -253,17 +252,16 @@ static void destroy_compound_page(struct page *page, unsigned long order)
        if (unlikely(compound_order(page) != order))
                bad_page(page);
 
-       if (unlikely(!PageCompound(page)))
+       if (unlikely(!PageHead(page)))
                        bad_page(page);
-       __ClearPageCompound(page);
+       __ClearPageHead(page);
        for (i = 1; i < nr_pages; i++) {
                struct page *p = page + i;
 
-               if (unlikely(!PageCompound(p) | !PageTail(p) |
+               if (unlikely(!PageTail(p) |
                                (p->first_page != page)))
                        bad_page(page);
                __ClearPageTail(p);
-               __ClearPageCompound(p);
        }
 }