From: Christoph Lameter Date: Tue, 16 Oct 2007 08:26:07 +0000 (-0700) Subject: SLUB: Avoid touching page struct when freeing to per cpu slab X-Git-Tag: v2.6.24-rc1~1121 X-Git-Url: http://pilppa.com/gitweb/?a=commitdiff_plain;h=ee3c72a14bfecdf783738032ff3c73ef6412f5b3;p=linux-2.6-omap-h63xx.git SLUB: Avoid touching page struct when freeing to per cpu slab Set c->node to -1 if we allocate from a debug slab instead for SlabDebug which requires access the page struct cacheline. Signed-off-by: Christoph Lameter Tested-by: Alexey Dobriyan Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- diff --git a/mm/slub.c b/mm/slub.c index 5d895d44c32..ea9fd72093d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -1537,6 +1537,7 @@ debug: c->page->inuse++; c->page->freelist = object[c->offset]; + c->node = -1; slab_unlock(c->page); return object; } @@ -1560,8 +1561,7 @@ static void __always_inline *slab_alloc(struct kmem_cache *s, local_irq_save(flags); c = get_cpu_slab(s, smp_processor_id()); - if (unlikely(!c->page || !c->freelist || - !node_match(c, node))) + if (unlikely(!c->freelist || !node_match(c, node))) object = __slab_alloc(s, gfpflags, node, addr, c); @@ -1670,7 +1670,7 @@ static void __always_inline slab_free(struct kmem_cache *s, local_irq_save(flags); debug_check_no_locks_freed(object, s->objsize); c = get_cpu_slab(s, smp_processor_id()); - if (likely(page == c->page && !SlabDebug(page))) { + if (likely(page == c->page && c->node >= 0)) { object[c->offset] = c->freelist; c->freelist = object; } else @@ -3250,12 +3250,16 @@ static unsigned long slab_objects(struct kmem_cache *s, for_each_possible_cpu(cpu) { struct page *page; + int node; struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); if (!c) continue; page = c->page; + node = c->node; + if (node < 0) + continue; if (page) { if (flags & SO_CPU) { int x = 0; @@ -3265,9 +3269,9 @@ static unsigned long slab_objects(struct kmem_cache *s, else x = 1; total += x; - nodes[c->node] += x; + nodes[node] += x; } - per_cpu[c->node]++; + per_cpu[node]++; } }