x86: track memtype for RAM in page struct

Track the memtype for RAM pages in page struct instead of using the
memtype list. This avoids the explosion in the number of entries in
memtype list (of the order of 20,000 with AGP) and makes the PAT
tracking simpler.

We are using PG_arch_1 bit in page->flags.

We still use the memtype list for non RAM pages.

Signed-off-by: Suresh Siddha <suresh.b.siddha@intel.com>
Signed-off-by: Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index d4b6e6a29ae3597c45071805f734a502a12d6ab2..d03c461e045e7402135c2bb086e6e5d040362f3f 100644 (file)
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -83,6 +83,25 @@ int page_is_ram(unsigned long pagenr)
        return 0;
 }
 
+int pagerange_is_ram(unsigned long start, unsigned long end)
+{
+       int ram_page = 0, not_rampage = 0;
+       unsigned long page_nr;
+
+       for (page_nr = (start >> PAGE_SHIFT); page_nr < (end >> PAGE_SHIFT);
+            ++page_nr) {
+               if (page_is_ram(page_nr))
+                       ram_page = 1;
+               else
+                       not_rampage = 1;
+
+               if (ram_page == not_rampage)
+                       return -1;
+       }
+
+       return ram_page;
+}
+
 /*
  * Fix up the linear direct mapping of the kernel to avoid cache attribute
  * conflicts.

diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index f049b1d6ebdfaff92ae7f11d1d8123a2b82a3f86..aceb6c7c6dba4dde5e9409a70355012951278c47 100644 (file)
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -210,6 +210,75 @@ static int chk_conflict(struct memtype *new, struct memtype *entry,
 static struct memtype *cached_entry;
 static u64 cached_start;
 
+/*
+ * RED-PEN:  TODO: Add PageReserved() check as well here,
+ * once we add SetPageReserved() to all the drivers using
+ * set_memory_* or set_pages_*.
+ *
+ * This will help prevent accidentally freeing pages
+ * before setting the attribute back to WB.
+ */
+
+/*
+ * For RAM pages, mark the pages as non WB memory type using
+ * PageNonWB (PG_arch_1). We allow only one set_memory_uc() or
+ * set_memory_wc() on a RAM page at a time before marking it as WB again.
+ * This is ok, because only one driver will be owning the page and
+ * doing set_memory_*() calls.
+ *
+ * For now, we use PageNonWB to track that the RAM page is being mapped
+ * as non WB. In future, we will have to use one more flag
+ * (or some other mechanism in page_struct) to distinguish between
+ * UC and WC mapping.
+ */
+static int reserve_ram_pages_type(u64 start, u64 end, unsigned long req_type,
+                      unsigned long *new_type)
+{
+       struct page *page;
+       u64 pfn, end_pfn;
+
+       for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
+               page = pfn_to_page(pfn);
+               if (page_mapped(page) || PageNonWB(page))
+                       goto out;
+
+               SetPageNonWB(page);
+       }
+       return 0;
+
+out:
+       end_pfn = pfn;
+       for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
+               page = pfn_to_page(pfn);
+               ClearPageNonWB(page);
+       }
+
+       return -EINVAL;
+}
+
+static int free_ram_pages_type(u64 start, u64 end)
+{
+       struct page *page;
+       u64 pfn, end_pfn;
+
+       for (pfn = (start >> PAGE_SHIFT); pfn < (end >> PAGE_SHIFT); ++pfn) {
+               page = pfn_to_page(pfn);
+               if (page_mapped(page) || !PageNonWB(page))
+                       goto out;
+
+               ClearPageNonWB(page);
+       }
+       return 0;
+
+out:
+       end_pfn = pfn;
+       for (pfn = (start >> PAGE_SHIFT); pfn < end_pfn; ++pfn) {
+               page = pfn_to_page(pfn);
+               SetPageNonWB(page);
+       }
+       return -EINVAL;
+}
+
 /*
  * req_type typically has one of the:
  * - _PAGE_CACHE_WB
@@ -232,6 +301,7 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
        unsigned long actual_type;
        struct list_head *where;
        int err = 0;
+       int is_range_ram;
 
        BUG_ON(start >= end); /* end is exclusive */
 
@@ -270,6 +340,12 @@ int reserve_memtype(u64 start, u64 end, unsigned long req_type,
                actual_type = pat_x_mtrr_type(start, end,
                                              req_type & _PAGE_CACHE_MASK);
 
+       is_range_ram = pagerange_is_ram(start, end);
+       if (is_range_ram == 1)
+               return reserve_ram_pages_type(start, end, req_type, new_type);
+       else if (is_range_ram < 0)
+               return -EINVAL;
+
        new  = kmalloc(sizeof(struct memtype), GFP_KERNEL);
        if (!new)
                return -ENOMEM;
@@ -358,6 +434,7 @@ int free_memtype(u64 start, u64 end)
 {
        struct memtype *entry;
        int err = -EINVAL;
+       int is_range_ram;
 
        if (!pat_enabled)
                return 0;
@@ -366,6 +443,12 @@ int free_memtype(u64 start, u64 end)
        if (is_ISA_range(start, end - 1))
                return 0;
 
+       is_range_ram = pagerange_is_ram(start, end);
+       if (is_range_ram == 1)
+               return free_ram_pages_type(start, end);
+       else if (is_range_ram < 0)
+               return -EINVAL;
+
        spin_lock(&memtype_lock);
        list_for_each_entry(entry, &memtype_list, nd) {
                if (entry->start == start && entry->end == end) {

diff --git a/include/asm-x86/cacheflush.h b/include/asm-x86/cacheflush.h
index 8e205a131250018d2f876177dabeebe5533a7954..092b9b4eb00301d765aa385c3f21c74c5eb80807 100644 (file)
--- a/include/asm-x86/cacheflush.h
+++ b/include/asm-x86/cacheflush.h
@@ -24,6 +24,8 @@
 #define copy_from_user_page(vma, page, vaddr, dst, src, len)   \
        memcpy((dst), (src), (len))
 
+#define PG_non_WB                              PG_arch_1
+PAGEFLAG(NonWB, non_WB)
 
 /*
  * The set_memory_* API can be used to change various attributes of a virtual

diff --git a/include/asm-x86/page.h b/include/asm-x86/page.h
index 49982110e4d94e4025896a6c05d7fddecca382ae..3407ac12ba34e6f20cfeddb3aff796f91aabe578 100644 (file)
--- a/include/asm-x86/page.h
+++ b/include/asm-x86/page.h
@@ -57,6 +57,7 @@ typedef struct { pgdval_t pgd; } pgd_t;
 typedef struct { pgprotval_t pgprot; } pgprot_t;
 
 extern int page_is_ram(unsigned long pagenr);
+extern int pagerange_is_ram(unsigned long start, unsigned long end);
 extern int devmem_is_allowed(unsigned long pagenr);
 extern void map_devmem(unsigned long pfn, unsigned long size,
                       pgprot_t vma_prot);