x86: introduce page pool in cpa

DEBUG_PAGEALLOC was not possible on 64-bit due to its early-bootup
hardcoded reliance on PSE pages, and the unrobustness of the runtime
splitup of large pages. The splitup ended in recursive calls to
alloc_pages() when a page for a pte split was requested.

Avoid the recursion with a preallocated page pool, which is used to
split up large mappings and gets refilled in the return path of
kernel_map_pages after the split has been done. The size of the page
pool is adjusted to the available memory.

This part just implements the page pool and the initialization w/o
using it yet.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/mm/init_32.c b/arch/x86/mm/init_32.c
index 54aba3cf9efe2c9a6ed85dfceb0adef199eb2e1e..8106bba41ecb4b7ce5b87a58545c50102cc562b7 100644 (file)
--- a/arch/x86/mm/init_32.c
+++ b/arch/x86/mm/init_32.c
@@ -664,6 +664,8 @@ void __init mem_init(void)
        if (boot_cpu_data.wp_works_ok < 0)
                test_wp_bit();
 
+       cpa_init();
+
        /*
         * Subtle. SMP is doing it's boot stuff late (because it has to
         * fork idle threads) - but it also needs low mappings for the

diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index 620d2b6b6bf42c74dfd9d760697865cc3d3e0957..b59fc238151fb58650f64c411c961511e725ee81 100644 (file)
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -528,6 +528,8 @@ void __init mem_init(void)
                reservedpages << (PAGE_SHIFT-10),
                datasize >> 10,
                initsize >> 10);
+
+       cpa_init();
 }
 
 void free_init_pages(char *what, unsigned long begin, unsigned long end)

diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index eb2a54415a77a12c99d13bf8b03a5aa42c377bf6..831462c3bc3578e47c428de55d023ef9e62ae77c 100644 (file)
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -8,6 +8,7 @@
 #include <linux/sched.h>
 #include <linux/slab.h>
 #include <linux/mm.h>
+#include <linux/interrupt.h>
 
 #include <asm/e820.h>
 #include <asm/processor.h>
@@ -336,6 +337,77 @@ out_unlock:
        return do_split;
 }
 
+static LIST_HEAD(page_pool);
+static unsigned long pool_size, pool_pages, pool_low;
+static unsigned long pool_used, pool_failed, pool_refill;
+
+static void cpa_fill_pool(void)
+{
+       struct page *p;
+       gfp_t gfp = GFP_KERNEL;
+
+       /* Do not allocate from interrupt context */
+       if (in_irq() || irqs_disabled())
+               return;
+       /*
+        * Check unlocked. I does not matter when we have one more
+        * page in the pool. The bit lock avoids recursive pool
+        * allocations:
+        */
+       if (pool_pages >= pool_size || test_and_set_bit_lock(0, &pool_refill))
+               return;
+
+#ifdef CONFIG_DEBUG_PAGEALLOC
+       /*
+        * We could do:
+        * gfp = in_atomic() ? GFP_ATOMIC : GFP_KERNEL;
+        * but this fails on !PREEMPT kernels
+        */
+       gfp =  GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
+#endif
+
+       while (pool_pages < pool_size) {
+               p = alloc_pages(gfp, 0);
+               if (!p) {
+                       pool_failed++;
+                       break;
+               }
+               spin_lock_irq(&pgd_lock);
+               list_add(&p->lru, &page_pool);
+               pool_pages++;
+               spin_unlock_irq(&pgd_lock);
+       }
+       clear_bit_unlock(0, &pool_refill);
+}
+
+#define SHIFT_MB               (20 - PAGE_SHIFT)
+#define ROUND_MB_GB            ((1 << 10) - 1)
+#define SHIFT_MB_GB            10
+#define POOL_PAGES_PER_GB      16
+
+void __init cpa_init(void)
+{
+       struct sysinfo si;
+       unsigned long gb;
+
+       si_meminfo(&si);
+       /*
+        * Calculate the number of pool pages:
+        *
+        * Convert totalram (nr of pages) to MiB and round to the next
+        * GiB. Shift MiB to Gib and multiply the result by
+        * POOL_PAGES_PER_GB:
+        */
+       gb = ((si.totalram >> SHIFT_MB) + ROUND_MB_GB) >> SHIFT_MB_GB;
+       pool_size = POOL_PAGES_PER_GB * gb;
+       pool_low = pool_size;
+
+       cpa_fill_pool();
+       printk(KERN_DEBUG
+              "CPA: page pool initialized %lu of %lu pages preallocated\n",
+              pool_pages, pool_size);
+}
+
 static int split_large_page(pte_t *kpte, unsigned long address)
 {
        unsigned long flags, pfn, pfninc = 1;
@@ -600,7 +672,7 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages,
         * Check whether we really changed something:
         */
        if (!cpa.flushtlb)
-               return ret;
+               goto out;
 
        /*
         * No need to flush, when we did not set any of the caching
@@ -619,6 +691,8 @@ static int change_page_attr_set_clr(unsigned long addr, int numpages,
        else
                cpa_flush_all(cache);
 
+out:
+       cpa_fill_pool();
        return ret;
 }
 
@@ -772,6 +846,12 @@ void kernel_map_pages(struct page *page, int numpages, int enable)
         * but that can deadlock->flush only current cpu:
         */
        __flush_tlb_all();
+
+       /*
+        * Try to refill the page pool here. We can do this only after
+        * the tlb flush.
+        */
+       cpa_fill_pool();
 }
 #endif
 

diff --git a/include/asm-x86/cacheflush.h b/include/asm-x86/cacheflush.h
index 8dd8c5e3cc7fca8dabc802662d1acd0263315366..6a22212b4b204b93570b84a600ec1e800e821aee 100644 (file)
--- a/include/asm-x86/cacheflush.h
+++ b/include/asm-x86/cacheflush.h
@@ -44,6 +44,8 @@ int set_memory_np(unsigned long addr, int numpages);
 
 void clflush_cache_range(void *addr, unsigned int size);
 
+void cpa_init(void);
+
 #ifdef CONFIG_DEBUG_RODATA
 void mark_rodata_ro(void);
 #endif