x86: support GB hugepages on 64-bit

Acked-by: Adam Litke <agl@us.ibm.com>
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Nick Piggin <npiggin@suse.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/arch/x86/mm/hugetlbpage.c b/arch/x86/mm/hugetlbpage.c
index a4789e87a31556de14895457b89cca24c49c88d7..b7a65a07af033ffea28f101f5a9cf4c6cc19f18e 100644 (file)
--- a/arch/x86/mm/hugetlbpage.c
+++ b/arch/x86/mm/hugetlbpage.c
@@ -134,9 +134,14 @@ pte_t *huge_pte_alloc(struct mm_struct *mm,
        pgd = pgd_offset(mm, addr);
        pud = pud_alloc(mm, pgd, addr);
        if (pud) {
-               if (pud_none(*pud))
-                       huge_pmd_share(mm, addr, pud);
-               pte = (pte_t *) pmd_alloc(mm, pud, addr);
+               if (sz == PUD_SIZE) {
+                       pte = (pte_t *)pud;
+               } else {
+                       BUG_ON(sz != PMD_SIZE);
+                       if (pud_none(*pud))
+                               huge_pmd_share(mm, addr, pud);
+                       pte = (pte_t *) pmd_alloc(mm, pud, addr);
+               }
        }
        BUG_ON(pte && !pte_none(*pte) && !pte_huge(*pte));
 
@@ -152,8 +157,11 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
        pgd = pgd_offset(mm, addr);
        if (pgd_present(*pgd)) {
                pud = pud_offset(pgd, addr);
-               if (pud_present(*pud))
+               if (pud_present(*pud)) {
+                       if (pud_large(*pud))
+                               return (pte_t *)pud;
                        pmd = pmd_offset(pud, addr);
+               }
        }
        return (pte_t *) pmd;
 }
@@ -216,7 +224,7 @@ int pmd_huge(pmd_t pmd)
 
 int pud_huge(pud_t pud)
 {
-       return 0;
+       return !!(pud_val(pud) & _PAGE_PSE);
 }
 
 struct page *
@@ -252,6 +260,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
                unsigned long addr, unsigned long len,
                unsigned long pgoff, unsigned long flags)
 {
+       struct hstate *h = hstate_file(file);
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
        unsigned long start_addr;
@@ -264,7 +273,7 @@ static unsigned long hugetlb_get_unmapped_area_bottomup(struct file *file,
        }
 
 full_search:
-       addr = ALIGN(start_addr, HPAGE_SIZE);
+       addr = ALIGN(start_addr, huge_page_size(h));
 
        for (vma = find_vma(mm, addr); ; vma = vma->vm_next) {
                /* At this point:  (!vma || addr < vma->vm_end). */
@@ -286,7 +295,7 @@ full_search:
                }
                if (addr + mm->cached_hole_size < vma->vm_start)
                        mm->cached_hole_size = vma->vm_start - addr;
-               addr = ALIGN(vma->vm_end, HPAGE_SIZE);
+               addr = ALIGN(vma->vm_end, huge_page_size(h));
        }
 }
 
@@ -294,6 +303,7 @@ static unsigned long hugetlb_get_unmapped_area_topdown(struct file *file,
                unsigned long addr0, unsigned long len,
                unsigned long pgoff, unsigned long flags)
 {
+       struct hstate *h = hstate_file(file);
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma, *prev_vma;
        unsigned long base = mm->mmap_base, addr = addr0;
@@ -314,7 +324,7 @@ try_again:
                goto fail;
 
        /* either no address requested or cant fit in requested address hole */
-       addr = (mm->free_area_cache - len) & HPAGE_MASK;
+       addr = (mm->free_area_cache - len) & huge_page_mask(h);
        do {
                /*
                 * Lookup failure means no vma is above this address,
@@ -345,7 +355,7 @@ try_again:
                        largest_hole = vma->vm_start - addr;
 
                /* try just below the current vma->vm_start */
-               addr = (vma->vm_start - len) & HPAGE_MASK;
+               addr = (vma->vm_start - len) & huge_page_mask(h);
        } while (len <= vma->vm_start);
 
 fail:
@@ -383,10 +393,11 @@ unsigned long
 hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
                unsigned long len, unsigned long pgoff, unsigned long flags)
 {
+       struct hstate *h = hstate_file(file);
        struct mm_struct *mm = current->mm;
        struct vm_area_struct *vma;
 
-       if (len & ~HPAGE_MASK)
+       if (len & ~huge_page_mask(h))
                return -EINVAL;
        if (len > TASK_SIZE)
                return -ENOMEM;
@@ -398,7 +409,7 @@ hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
        }
 
        if (addr) {
-               addr = ALIGN(addr, HPAGE_SIZE);
+               addr = ALIGN(addr, huge_page_size(h));
                vma = find_vma(mm, addr);
                if (TASK_SIZE - len >= addr &&
                    (!vma || addr + len <= vma->vm_start))