powerpc/mm: Rework usage of _PAGE_COHERENT/NO_CACHE/GUARDED

Currently, we never set _PAGE_COHERENT in the PTEs, we just OR it in
in the hash code based on some CPU feature bit.  We also manipulate
_PAGE_NO_CACHE and _PAGE_GUARDED by hand in all sorts of places.

This changes the logic so that instead, the PTE now contains
_PAGE_COHERENT for all normal RAM pages thay have I = 0 on platforms
that need it.  The hash code clears it if the feature bit is not set.

It also adds some clean accessors to setup various valid combinations
of access flags and change various bits of code to use them instead.

This should help having the PTE actually containing the bit
combinations that we really want.

I also removed _PAGE_GUARDED from _PAGE_BASE on 44x and instead
set it explicitely from the TLB miss.  I will ultimately remove it
completely as it appears that it might not be needed after all
but in the meantime, having it in the TLB miss makes things a
lot easier.

Signed-off-by: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Acked-by: Kumar Gala <galak@kernel.crashing.org>
Signed-off-by: Paul Mackerras <paulus@samba.org>

diff --git a/arch/powerpc/include/asm/pgtable-ppc32.h b/arch/powerpc/include/asm/pgtable-ppc32.h
index 6ab7c67cb5ab6355ba453e4832ad55bcdb03ec85..f69a4d97772904ea5c42beda4748a8d4cc085202 100644 (file)
--- a/arch/powerpc/include/asm/pgtable-ppc32.h
+++ b/arch/powerpc/include/asm/pgtable-ppc32.h
@@ -228,9 +228,10 @@ extern int icache_44x_need_flush;
  *   - FILE *must* be in the bottom three bits because swap cache
  *     entries use the top 29 bits for TLB2.
  *
- *   - CACHE COHERENT bit (M) has no effect on PPC440 core, because it
- *     doesn't support SMP. So we can use this as software bit, like
- *     DIRTY.
+ *   - CACHE COHERENT bit (M) has no effect on original PPC440 cores,
+ *     because it doesn't support SMP. However, some later 460 variants
+ *     have -some- form of SMP support and so I keep the bit there for
+ *     future use
  *
  * With the PPC 44x Linux implementation, the 0-11th LSBs of the PTE are used
  * for memory protection related functions (see PTE structure in
@@ -436,20 +437,23 @@ extern int icache_44x_need_flush;
                         _PAGE_USER | _PAGE_ACCESSED | \
                         _PAGE_RW | _PAGE_HWWRITE | _PAGE_DIRTY | \
                         _PAGE_EXEC | _PAGE_HWEXEC)
+
 /*
- * Note: the _PAGE_COHERENT bit automatically gets set in the hardware
- * PTE if CONFIG_SMP is defined (hash_page does this); there is no need
- * to have it in the Linux PTE, and in fact the bit could be reused for
- * another purpose.  -- paulus.
+ * We define 2 sets of base prot bits, one for basic pages (ie,
+ * cacheable kernel and user pages) and one for non cacheable
+ * pages. We always set _PAGE_COHERENT when SMP is enabled or
+ * the processor might need it for DMA coherency.
  */
-
-#ifdef CONFIG_44x
-#define _PAGE_BASE     (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_GUARDED)
+#if defined(CONFIG_SMP) || defined(CONFIG_PPC_STD_MMU)
+#define _PAGE_BASE     (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_COHERENT)
 #else
 #define _PAGE_BASE     (_PAGE_PRESENT | _PAGE_ACCESSED)
 #endif
+#define _PAGE_BASE_NC  (_PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_NO_CACHE)
+
 #define _PAGE_WRENABLE (_PAGE_RW | _PAGE_DIRTY | _PAGE_HWWRITE)
 #define _PAGE_KERNEL   (_PAGE_BASE | _PAGE_SHARED | _PAGE_WRENABLE)
+#define _PAGE_KERNEL_NC        (_PAGE_BASE_NC | _PAGE_SHARED | _PAGE_WRENABLE)
 
 #ifdef CONFIG_PPC_STD_MMU
 /* On standard PPC MMU, no user access implies kernel read/write access,
@@ -459,7 +463,7 @@ extern int icache_44x_need_flush;
 #define _PAGE_KERNEL_RO        (_PAGE_BASE | _PAGE_SHARED)
 #endif
 
-#define _PAGE_IO       (_PAGE_KERNEL | _PAGE_NO_CACHE | _PAGE_GUARDED)
+#define _PAGE_IO       (_PAGE_KERNEL_NC | _PAGE_GUARDED)
 #define _PAGE_RAM      (_PAGE_KERNEL | _PAGE_HWEXEC)
 
 #if defined(CONFIG_KGDB) || defined(CONFIG_XMON) || defined(CONFIG_BDI_SWITCH) ||\
@@ -552,9 +556,6 @@ static inline int pte_young(pte_t pte)              { return pte_val(pte) & _PAGE_ACCESSED;
 static inline int pte_file(pte_t pte)          { return pte_val(pte) & _PAGE_FILE; }
 static inline int pte_special(pte_t pte)       { return pte_val(pte) & _PAGE_SPECIAL; }
 
-static inline void pte_uncache(pte_t pte)       { pte_val(pte) |= _PAGE_NO_CACHE; }
-static inline void pte_cache(pte_t pte)         { pte_val(pte) &= ~_PAGE_NO_CACHE; }
-
 static inline pte_t pte_wrprotect(pte_t pte) {
        pte_val(pte) &= ~(_PAGE_RW | _PAGE_HWWRITE); return pte; }
 static inline pte_t pte_mkclean(pte_t pte) {
@@ -693,10 +694,11 @@ static inline void __set_pte_at(struct mm_struct *mm, unsigned long addr,
 #endif
 }
 
+
 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
                              pte_t *ptep, pte_t pte)
 {
-#if defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP)
+#if defined(CONFIG_PTE_64BIT) && defined(CONFIG_SMP) && defined(CONFIG_DEBUG_VM)
        WARN_ON(pte_present(*ptep));
 #endif
        __set_pte_at(mm, addr, ptep, pte);
@@ -760,16 +762,6 @@ static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
        __changed;                                                         \
 })
 
-/*
- * Macro to mark a page protection value as "uncacheable".
- */
-#define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_NO_CACHE | _PAGE_GUARDED))
-
-struct file;
-extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
-                                    unsigned long size, pgprot_t vma_prot);
-#define __HAVE_PHYS_MEM_ACCESS_PROT
-
 #define __HAVE_ARCH_PTE_SAME
 #define pte_same(A,B)  (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HASHPTE) == 0)
 

diff --git a/arch/powerpc/include/asm/pgtable-ppc64.h b/arch/powerpc/include/asm/pgtable-ppc64.h
index 1f0a330f03f498f0076547c28e5735d822e8935b..b0f18be81d9fbde440a17ccf52e785cfb3b7a408 100644 (file)
--- a/arch/powerpc/include/asm/pgtable-ppc64.h
+++ b/arch/powerpc/include/asm/pgtable-ppc64.h
@@ -245,9 +245,6 @@ static inline int pte_young(pte_t pte) { return pte_val(pte) & _PAGE_ACCESSED;}
 static inline int pte_file(pte_t pte) { return pte_val(pte) & _PAGE_FILE;}
 static inline int pte_special(pte_t pte) { return pte_val(pte) & _PAGE_SPECIAL; }
 
-static inline void pte_uncache(pte_t pte) { pte_val(pte) |= _PAGE_NO_CACHE; }
-static inline void pte_cache(pte_t pte)   { pte_val(pte) &= ~_PAGE_NO_CACHE; }
-
 static inline pte_t pte_wrprotect(pte_t pte) {
        pte_val(pte) &= ~(_PAGE_RW); return pte; }
 static inline pte_t pte_mkclean(pte_t pte) {
@@ -405,16 +402,6 @@ static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry, int dirty)
        __changed;                                                         \
 })
 
-/*
- * Macro to mark a page protection value as "uncacheable".
- */
-#define pgprot_noncached(prot) (__pgprot(pgprot_val(prot) | _PAGE_NO_CACHE | _PAGE_GUARDED))
-
-struct file;
-extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
-                                    unsigned long size, pgprot_t vma_prot);
-#define __HAVE_PHYS_MEM_ACCESS_PROT
-
 #define __HAVE_ARCH_PTE_SAME
 #define pte_same(A,B)  (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0)
 

diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index dbb8ca172e445461a8c61bcad3a994bf62276f96..07f55e6016962c4af3c9acc34fd9b54f04f44a8b 100644 (file)
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -16,6 +16,32 @@ struct mm_struct;
 #endif
 
 #ifndef __ASSEMBLY__
+
+/*
+ * Macro to mark a page protection value as "uncacheable".
+ */
+
+#define _PAGE_CACHE_CTL        (_PAGE_COHERENT | _PAGE_GUARDED | _PAGE_NO_CACHE | \
+                        _PAGE_WRITETHRU)
+
+#define pgprot_noncached(prot)   (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
+                                           _PAGE_NO_CACHE | _PAGE_GUARDED))
+
+#define pgprot_noncached_wc(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
+                                           _PAGE_NO_CACHE))
+
+#define pgprot_cached(prot)       (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
+                                           _PAGE_COHERENT))
+
+#define pgprot_cached_wthru(prot) (__pgprot((pgprot_val(prot) & ~_PAGE_CACHE_CTL) | \
+                                           _PAGE_COHERENT | _PAGE_WRITETHRU))
+
+
+struct file;
+extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
+                                    unsigned long size, pgprot_t vma_prot);
+#define __HAVE_PHYS_MEM_ACCESS_PROT
+
 /*
  * ZERO_PAGE is a global shared page that is always zero: used
  * for zero-mapped memory areas etc..

diff --git a/arch/powerpc/kernel/head_44x.S b/arch/powerpc/kernel/head_44x.S
index f3a1ea9d7fe44d8e45371d3f9b91863277fa4756..26237357a88c9bbb1881db3688c26d478a65fd64 100644 (file)
--- a/arch/powerpc/kernel/head_44x.S
+++ b/arch/powerpc/kernel/head_44x.S
@@ -570,6 +570,7 @@ finish_tlb_load:
        rlwimi  r10,r12,29,30,30                /* DIRTY -> SW position */
        and     r11,r12,r10                     /* Mask PTE bits to keep */
        andi.   r10,r12,_PAGE_USER              /* User page ? */
+       ori     r11,r11,_PAGE_GUARDED           /* 440 errata, needs G set */
        beq     1f                              /* nope, leave U bits empty */
        rlwimi  r11,r11,3,26,28                 /* yes, copy S bits to U */
 1:     tlbwe   r11,r13,PPC44x_TLB_ATTRIB       /* Write ATTRIB */

diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c
index 1a32db331a5c0aa576d04d717c998ae2b460f367..2538030954d85102c3ac929b076ad3979d98a4c1 100644 (file)
--- a/arch/powerpc/kernel/pci-common.c
+++ b/arch/powerpc/kernel/pci-common.c
@@ -370,13 +370,10 @@ static pgprot_t __pci_mmap_set_pgprot(struct pci_dev *dev, struct resource *rp,
        }
 
        /* XXX would be nice to have a way to ask for write-through */
-       prot |= _PAGE_NO_CACHE;
        if (write_combine)
-               prot &= ~_PAGE_GUARDED;
+               return pgprot_noncached_wc(prot);
        else
-               prot |= _PAGE_GUARDED;
-
-       return __pgprot(prot);
+               return pgprot_noncached(prot);
 }
 
 /*
@@ -387,19 +384,17 @@ static pgprot_t __pci_mmap_set_pgprot(struct pci_dev *dev, struct resource *rp,
 pgprot_t pci_phys_mem_access_prot(struct file *file,
                                  unsigned long pfn,
                                  unsigned long size,
-                                 pgprot_t protection)
+                                 pgprot_t prot)
 {
        struct pci_dev *pdev = NULL;
        struct resource *found = NULL;
-       unsigned long prot = pgprot_val(protection);
        resource_size_t offset = ((resource_size_t)pfn) << PAGE_SHIFT;
        int i;
 
        if (page_is_ram(pfn))
-               return __pgprot(prot);
-
-       prot |= _PAGE_NO_CACHE | _PAGE_GUARDED;
+               return prot;
 
+       prot = pgprot_noncached(prot);
        for_each_pci_dev(pdev) {
                for (i = 0; i <= PCI_ROM_RESOURCE; i++) {
                        struct resource *rp = &pdev->resource[i];
@@ -420,14 +415,14 @@ pgprot_t pci_phys_mem_access_prot(struct file *file,
        }
        if (found) {
                if (found->flags & IORESOURCE_PREFETCH)
-                       prot &= ~_PAGE_GUARDED;
+                       prot = pgprot_noncached_wc(prot);
                pci_dev_put(pdev);
        }
 
        pr_debug("PCI: Non-PCI map for %llx, prot: %lx\n",
-                (unsigned long long)offset, prot);
+                (unsigned long long)offset, pgprot_val(prot));
 
-       return __pgprot(prot);
+       return prot;
 }
 
 
@@ -583,8 +578,7 @@ int pci_mmap_legacy_page_range(struct pci_bus *bus,
        pr_debug(" -> mapping phys %llx\n", (unsigned long long)offset);
 
        vma->vm_pgoff = offset >> PAGE_SHIFT;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
        return remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
                               vma->vm_end - vma->vm_start,
                               vma->vm_page_prot);

diff --git a/arch/powerpc/mm/hash_low_32.S b/arch/powerpc/mm/hash_low_32.S
index c8eac22a8f000164adeb4446a90714de47a864c7..28845604a10c4fc34e2629e7aabaadefbecdaf2b 100644 (file)
--- a/arch/powerpc/mm/hash_low_32.S
+++ b/arch/powerpc/mm/hash_low_32.S
@@ -323,8 +323,8 @@ _GLOBAL(create_hpte)
        ori     r8,r8,0xe14             /* clear out reserved bits and M */
        andc    r8,r5,r8                /* PP = user? (rw&dirty? 2: 3): 0 */
 BEGIN_FTR_SECTION
-       ori     r8,r8,_PAGE_COHERENT    /* set M (coherence required) */
-END_FTR_SECTION_IFSET(CPU_FTR_NEED_COHERENT)
+       rlwinm  r8,r8,0,~_PAGE_COHERENT /* clear M (coherence not required) */
+END_FTR_SECTION_IFCLR(CPU_FTR_NEED_COHERENT)
 #ifdef CONFIG_PTE_64BIT
        /* Put the XPN bits into the PTE */
        rlwimi  r8,r10,8,20,22

diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c
index 8fee696fb79588eb40b9c31661f65dee9b22a4c7..53b06ebb3f2f7cad0a9dd690f30d0582184e8042 100644 (file)
--- a/arch/powerpc/mm/mem.c
+++ b/arch/powerpc/mm/mem.c
@@ -102,8 +102,8 @@ pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
                return ppc_md.phys_mem_access_prot(file, pfn, size, vma_prot);
 
        if (!page_is_ram(pfn))
-               vma_prot = __pgprot(pgprot_val(vma_prot)
-                                   | _PAGE_GUARDED | _PAGE_NO_CACHE);
+               vma_prot = pgprot_noncached(vma_prot);
+
        return vma_prot;
 }
 EXPORT_SYMBOL(phys_mem_access_prot);

diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c
index 1b26071a86ca8f17e1f2c04f22ffb9d60d13e2b8..7106b63d401bce3dbe5f3681c293a845f54c190c 100644 (file)
--- a/arch/powerpc/platforms/cell/spufs/file.c
+++ b/arch/powerpc/platforms/cell/spufs/file.c
@@ -273,12 +273,10 @@ spufs_mem_mmap_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
                return VM_FAULT_NOPAGE;
 
        if (ctx->state == SPU_STATE_SAVED) {
-               vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                                       & ~_PAGE_NO_CACHE);
+               vma->vm_page_prot = pgprot_cached(vma->vm_page_prot);
                pfn = vmalloc_to_pfn(ctx->csa.lscsa->ls + offset);
        } else {
-               vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                            | _PAGE_NO_CACHE);
+               vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
                pfn = (ctx->spu->local_store_phys + offset) >> PAGE_SHIFT;
        }
        vm_insert_pfn(vma, address, pfn);
@@ -338,8 +336,7 @@ static int spufs_mem_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE);
+       vma->vm_page_prot = pgprot_noncached_wc(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_mem_mmap_vmops;
        return 0;
@@ -452,8 +449,7 @@ static int spufs_cntl_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_cntl_mmap_vmops;
        return 0;
@@ -1155,8 +1151,7 @@ static int spufs_signal1_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_signal1_mmap_vmops;
        return 0;
@@ -1292,8 +1287,7 @@ static int spufs_signal2_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_signal2_mmap_vmops;
        return 0;
@@ -1414,8 +1408,7 @@ static int spufs_mss_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_mss_mmap_vmops;
        return 0;
@@ -1476,8 +1469,7 @@ static int spufs_psmap_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_psmap_mmap_vmops;
        return 0;
@@ -1536,8 +1528,7 @@ static int spufs_mfc_mmap(struct file *file, struct vm_area_struct *vma)
                return -EINVAL;
 
        vma->vm_flags |= VM_IO | VM_PFNMAP;
-       vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
-                                    | _PAGE_NO_CACHE | _PAGE_GUARDED);
+       vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 
        vma->vm_ops = &spufs_mfc_mmap_vmops;
        return 0;

diff --git a/drivers/video/controlfb.c b/drivers/video/controlfb.c
index b0be7eac32d817c8cec0255374ae80ab87483d87..49fcbe8f18ac4dbbd958f5e1973be85dda2875ae 100644 (file)
--- a/drivers/video/controlfb.c
+++ b/drivers/video/controlfb.c
@@ -298,10 +298,10 @@ static int controlfb_mmap(struct fb_info *info,
                        return -EINVAL;
                start = info->fix.mmio_start;
                len = PAGE_ALIGN((start & ~PAGE_MASK)+info->fix.mmio_len);
-               pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE|_PAGE_GUARDED;
+              vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
        } else {
                /* framebuffer */
-               pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU;
+              vma->vm_page_prot = pgprot_cached_wthru(vma->vm_page_prot);
        }
        start &= PAGE_MASK;
        if ((vma->vm_end - vma->vm_start + off) > len)