[PATCH] slab: fix kmalloc_node applying memory policies if nodeid == numa_node_id()

kmalloc_node() falls back to ___cache_alloc() under certain conditions and
at that point memory policies may be applied redirecting the allocation
away from the current node.  Therefore kmalloc_node(...,numa_node_id()) or
kmalloc_node(...,-1) may not return memory from the local node.

Fix this by doing the policy check in __cache_alloc() instead of
____cache_alloc().

This version here is a cleanup of Kiran's patch.

- Tested on ia64.
- Extra material removed.
- Consolidate the exit path if alternate_node_alloc() returned an object.

[akpm@osdl.org: warning fix]
Signed-off-by: Alok N Kataria <alok.kataria@calsoftinc.com>
Signed-off-by: Ravikiran Thirumalai <kiran@scalex86.org>
Signed-off-by: Shai Fultheim <shai@scalex86.org>
Signed-off-by: Christoph Lameter <clameter@sgi.com>
Signed-off-by: Andrew Morton <akpm@osdl.org>
Signed-off-by: Linus Torvalds <torvalds@osdl.org>

diff --git a/mm/slab.c b/mm/slab.c
index c52ebf9c446242b193b0b59ad6357e96f3ba6fa6..69e11c45002fe01b628ca19b2d018426b124c435 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3028,14 +3028,6 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
        void *objp;
        struct array_cache *ac;
 
-#ifdef CONFIG_NUMA
-       if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY))) {
-               objp = alternate_node_alloc(cachep, flags);
-               if (objp != NULL)
-                       return objp;
-       }
-#endif
-
        check_irq_off();
        ac = cpu_cache_get(cachep);
        if (likely(ac->avail)) {
@@ -3053,12 +3045,19 @@ static __always_inline void *__cache_alloc(struct kmem_cache *cachep,
                                                gfp_t flags, void *caller)
 {
        unsigned long save_flags;
-       void *objp;
+       void *objp = NULL;
 
        cache_alloc_debugcheck_before(cachep, flags);
 
        local_irq_save(save_flags);
-       objp = ____cache_alloc(cachep, flags);
+
+#ifdef CONFIG_NUMA
+       if (unlikely(current->flags & (PF_SPREAD_SLAB | PF_MEMPOLICY)))
+               objp = alternate_node_alloc(cachep, flags);
+#endif
+
+       if (!objp)
+               objp = ____cache_alloc(cachep, flags);
        local_irq_restore(save_flags);
        objp = cache_alloc_debugcheck_after(cachep, flags, objp,
                                            caller);