Commit db5e53fb authored by Linus Torvalds's avatar Linus Torvalds

Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/penberg/slab-2.6:
  slub: avoid leaking caches or refcounts on sysfs error
  slab: Fix comment on #endif
  slab: remove GFP_THISNODE clearing from alloc_slabmgmt()
  slub: Add might_sleep_if() to slab_alloc()
  SLUB: failslab support
  slub: Fix incorrect use of loose
  slab: Update the kmem_cache_create documentation regarding the name parameter
  slub: make early_kmem_cache_node_alloc void
  slab: unsigned slabp->inuse cannot be less than 0
  slub - fix get_object_page comment
  SLUB: Replace __builtin_return_address(0) with _RET_IP_.
  SLUB: cleanup - define macros instead of hardcoded numbers
parents 3f4b5c5d 3c506efd
...@@ -81,4 +81,13 @@ static inline void cleanup_fault_attr_dentries(struct fault_attr *attr) ...@@ -81,4 +81,13 @@ static inline void cleanup_fault_attr_dentries(struct fault_attr *attr)
#endif /* CONFIG_FAULT_INJECTION */ #endif /* CONFIG_FAULT_INJECTION */
#ifdef CONFIG_FAILSLAB
extern bool should_failslab(size_t size, gfp_t gfpflags);
#else
static inline bool should_failslab(size_t size, gfp_t gfpflags)
{
return false;
}
#endif /* CONFIG_FAILSLAB */
#endif /* _LINUX_FAULT_INJECT_H */ #endif /* _LINUX_FAULT_INJECT_H */
...@@ -253,9 +253,9 @@ static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep, ...@@ -253,9 +253,9 @@ static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep,
* request comes from. * request comes from.
*/ */
#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB)
extern void *__kmalloc_track_caller(size_t, gfp_t, void*); extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long);
#define kmalloc_track_caller(size, flags) \ #define kmalloc_track_caller(size, flags) \
__kmalloc_track_caller(size, flags, __builtin_return_address(0)) __kmalloc_track_caller(size, flags, _RET_IP_)
#else #else
#define kmalloc_track_caller(size, flags) \ #define kmalloc_track_caller(size, flags) \
__kmalloc(size, flags) __kmalloc(size, flags)
...@@ -271,10 +271,10 @@ extern void *__kmalloc_track_caller(size_t, gfp_t, void*); ...@@ -271,10 +271,10 @@ extern void *__kmalloc_track_caller(size_t, gfp_t, void*);
* allocation request comes from. * allocation request comes from.
*/ */
#if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB) #if defined(CONFIG_DEBUG_SLAB) || defined(CONFIG_SLUB)
extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, void *); extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, unsigned long);
#define kmalloc_node_track_caller(size, flags, node) \ #define kmalloc_node_track_caller(size, flags, node) \
__kmalloc_node_track_caller(size, flags, node, \ __kmalloc_node_track_caller(size, flags, node, \
__builtin_return_address(0)) _RET_IP_)
#else #else
#define kmalloc_node_track_caller(size, flags, node) \ #define kmalloc_node_track_caller(size, flags, node) \
__kmalloc_node(size, flags, node) __kmalloc_node(size, flags, node)
...@@ -285,7 +285,7 @@ extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, void *); ...@@ -285,7 +285,7 @@ extern void *__kmalloc_node_track_caller(size_t, gfp_t, int, void *);
#define kmalloc_node_track_caller(size, flags, node) \ #define kmalloc_node_track_caller(size, flags, node) \
kmalloc_track_caller(size, flags) kmalloc_track_caller(size, flags)
#endif /* DEBUG_SLAB */ #endif /* CONFIG_NUMA */
/* /*
* Shortcuts * Shortcuts
......
...@@ -730,6 +730,7 @@ config FAULT_INJECTION ...@@ -730,6 +730,7 @@ config FAULT_INJECTION
config FAILSLAB config FAILSLAB
bool "Fault-injection capability for kmalloc" bool "Fault-injection capability for kmalloc"
depends on FAULT_INJECTION depends on FAULT_INJECTION
depends on SLAB || SLUB
help help
Provide fault-injection capability for kmalloc. Provide fault-injection capability for kmalloc.
......
...@@ -28,6 +28,7 @@ obj-$(CONFIG_SLOB) += slob.o ...@@ -28,6 +28,7 @@ obj-$(CONFIG_SLOB) += slob.o
obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o obj-$(CONFIG_MMU_NOTIFIER) += mmu_notifier.o
obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_SLAB) += slab.o
obj-$(CONFIG_SLUB) += slub.o obj-$(CONFIG_SLUB) += slub.o
obj-$(CONFIG_FAILSLAB) += failslab.o
obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_FS_XIP) += filemap_xip.o
obj-$(CONFIG_MIGRATION) += migrate.o obj-$(CONFIG_MIGRATION) += migrate.o
......
#include <linux/fault-inject.h>
static struct {
struct fault_attr attr;
u32 ignore_gfp_wait;
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
struct dentry *ignore_gfp_wait_file;
#endif
} failslab = {
.attr = FAULT_ATTR_INITIALIZER,
.ignore_gfp_wait = 1,
};
bool should_failslab(size_t size, gfp_t gfpflags)
{
if (gfpflags & __GFP_NOFAIL)
return false;
if (failslab.ignore_gfp_wait && (gfpflags & __GFP_WAIT))
return false;
return should_fail(&failslab.attr, size);
}
static int __init setup_failslab(char *str)
{
return setup_fault_attr(&failslab.attr, str);
}
__setup("failslab=", setup_failslab);
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
static int __init failslab_debugfs_init(void)
{
mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
struct dentry *dir;
int err;
err = init_fault_attr_dentries(&failslab.attr, "failslab");
if (err)
return err;
dir = failslab.attr.dentries.dir;
failslab.ignore_gfp_wait_file =
debugfs_create_bool("ignore-gfp-wait", mode, dir,
&failslab.ignore_gfp_wait);
if (!failslab.ignore_gfp_wait_file) {
err = -ENOMEM;
debugfs_remove(failslab.ignore_gfp_wait_file);
cleanup_fault_attr_dentries(&failslab.attr);
}
return err;
}
late_initcall(failslab_debugfs_init);
#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
...@@ -2123,6 +2123,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep) ...@@ -2123,6 +2123,8 @@ static int __init_refok setup_cpu_cache(struct kmem_cache *cachep)
* *
* @name must be valid until the cache is destroyed. This implies that * @name must be valid until the cache is destroyed. This implies that
* the module calling this has to destroy the cache before getting unloaded. * the module calling this has to destroy the cache before getting unloaded.
* Note that kmem_cache_name() is not guaranteed to return the same pointer,
* therefore applications must manage it themselves.
* *
* The flags are * The flags are
* *
...@@ -2609,7 +2611,7 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp, ...@@ -2609,7 +2611,7 @@ static struct slab *alloc_slabmgmt(struct kmem_cache *cachep, void *objp,
if (OFF_SLAB(cachep)) { if (OFF_SLAB(cachep)) {
/* Slab management obj is off-slab. */ /* Slab management obj is off-slab. */
slabp = kmem_cache_alloc_node(cachep->slabp_cache, slabp = kmem_cache_alloc_node(cachep->slabp_cache,
local_flags & ~GFP_THISNODE, nodeid); local_flags, nodeid);
if (!slabp) if (!slabp)
return NULL; return NULL;
} else { } else {
...@@ -2997,7 +2999,7 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags) ...@@ -2997,7 +2999,7 @@ static void *cache_alloc_refill(struct kmem_cache *cachep, gfp_t flags)
* there must be at least one object available for * there must be at least one object available for
* allocation. * allocation.
*/ */
BUG_ON(slabp->inuse < 0 || slabp->inuse >= cachep->num); BUG_ON(slabp->inuse >= cachep->num);
while (slabp->inuse < cachep->num && batchcount--) { while (slabp->inuse < cachep->num && batchcount--) {
STATS_INC_ALLOCED(cachep); STATS_INC_ALLOCED(cachep);
...@@ -3106,79 +3108,14 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep, ...@@ -3106,79 +3108,14 @@ static void *cache_alloc_debugcheck_after(struct kmem_cache *cachep,
#define cache_alloc_debugcheck_after(a,b,objp,d) (objp) #define cache_alloc_debugcheck_after(a,b,objp,d) (objp)
#endif #endif
#ifdef CONFIG_FAILSLAB static bool slab_should_failslab(struct kmem_cache *cachep, gfp_t flags)
static struct failslab_attr {
struct fault_attr attr;
u32 ignore_gfp_wait;
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
struct dentry *ignore_gfp_wait_file;
#endif
} failslab = {
.attr = FAULT_ATTR_INITIALIZER,
.ignore_gfp_wait = 1,
};
static int __init setup_failslab(char *str)
{
return setup_fault_attr(&failslab.attr, str);
}
__setup("failslab=", setup_failslab);
static int should_failslab(struct kmem_cache *cachep, gfp_t flags)
{ {
if (cachep == &cache_cache) if (cachep == &cache_cache)
return 0; return false;
if (flags & __GFP_NOFAIL)
return 0;
if (failslab.ignore_gfp_wait && (flags & __GFP_WAIT))
return 0;
return should_fail(&failslab.attr, obj_size(cachep)); return should_failslab(obj_size(cachep), flags);
} }
#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
static int __init failslab_debugfs(void)
{
mode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
struct dentry *dir;
int err;
err = init_fault_attr_dentries(&failslab.attr, "failslab");
if (err)
return err;
dir = failslab.attr.dentries.dir;
failslab.ignore_gfp_wait_file =
debugfs_create_bool("ignore-gfp-wait", mode, dir,
&failslab.ignore_gfp_wait);
if (!failslab.ignore_gfp_wait_file) {
err = -ENOMEM;
debugfs_remove(failslab.ignore_gfp_wait_file);
cleanup_fault_attr_dentries(&failslab.attr);
}
return err;
}
late_initcall(failslab_debugfs);
#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
#else /* CONFIG_FAILSLAB */
static inline int should_failslab(struct kmem_cache *cachep, gfp_t flags)
{
return 0;
}
#endif /* CONFIG_FAILSLAB */
static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags)
{ {
void *objp; void *objp;
...@@ -3381,7 +3318,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, ...@@ -3381,7 +3318,7 @@ __cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
unsigned long save_flags; unsigned long save_flags;
void *ptr; void *ptr;
if (should_failslab(cachep, flags)) if (slab_should_failslab(cachep, flags))
return NULL; return NULL;
cache_alloc_debugcheck_before(cachep, flags); cache_alloc_debugcheck_before(cachep, flags);
...@@ -3457,7 +3394,7 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller) ...@@ -3457,7 +3394,7 @@ __cache_alloc(struct kmem_cache *cachep, gfp_t flags, void *caller)
unsigned long save_flags; unsigned long save_flags;
void *objp; void *objp;
if (should_failslab(cachep, flags)) if (slab_should_failslab(cachep, flags))
return NULL; return NULL;
cache_alloc_debugcheck_before(cachep, flags); cache_alloc_debugcheck_before(cachep, flags);
...@@ -3686,9 +3623,9 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ...@@ -3686,9 +3623,9 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
EXPORT_SYMBOL(__kmalloc_node); EXPORT_SYMBOL(__kmalloc_node);
void *__kmalloc_node_track_caller(size_t size, gfp_t flags, void *__kmalloc_node_track_caller(size_t size, gfp_t flags,
int node, void *caller) int node, unsigned long caller)
{ {
return __do_kmalloc_node(size, flags, node, caller); return __do_kmalloc_node(size, flags, node, (void *)caller);
} }
EXPORT_SYMBOL(__kmalloc_node_track_caller); EXPORT_SYMBOL(__kmalloc_node_track_caller);
#else #else
...@@ -3730,9 +3667,9 @@ void *__kmalloc(size_t size, gfp_t flags) ...@@ -3730,9 +3667,9 @@ void *__kmalloc(size_t size, gfp_t flags)
} }
EXPORT_SYMBOL(__kmalloc); EXPORT_SYMBOL(__kmalloc);
void *__kmalloc_track_caller(size_t size, gfp_t flags, void *caller) void *__kmalloc_track_caller(size_t size, gfp_t flags, unsigned long caller)
{ {
return __do_kmalloc(size, flags, caller); return __do_kmalloc(size, flags, (void *)caller);
} }
EXPORT_SYMBOL(__kmalloc_track_caller); EXPORT_SYMBOL(__kmalloc_track_caller);
......
...@@ -24,6 +24,7 @@ ...@@ -24,6 +24,7 @@
#include <linux/kallsyms.h> #include <linux/kallsyms.h>
#include <linux/memory.h> #include <linux/memory.h>
#include <linux/math64.h> #include <linux/math64.h>
#include <linux/fault-inject.h>
/* /*
* Lock order: * Lock order:
...@@ -153,6 +154,10 @@ ...@@ -153,6 +154,10 @@
#define ARCH_SLAB_MINALIGN __alignof__(unsigned long long) #define ARCH_SLAB_MINALIGN __alignof__(unsigned long long)
#endif #endif
#define OO_SHIFT 16
#define OO_MASK ((1 << OO_SHIFT) - 1)
#define MAX_OBJS_PER_PAGE 65535 /* since page.objects is u16 */
/* Internal SLUB flags */ /* Internal SLUB flags */
#define __OBJECT_POISON 0x80000000 /* Poison object */ #define __OBJECT_POISON 0x80000000 /* Poison object */
#define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */ #define __SYSFS_ADD_DEFERRED 0x40000000 /* Not yet visible via sysfs */
...@@ -178,7 +183,7 @@ static LIST_HEAD(slab_caches); ...@@ -178,7 +183,7 @@ static LIST_HEAD(slab_caches);
* Tracking user of a slab. * Tracking user of a slab.
*/ */
struct track { struct track {
void *addr; /* Called from address */ unsigned long addr; /* Called from address */
int cpu; /* Was running on cpu */ int cpu; /* Was running on cpu */
int pid; /* Pid context */ int pid; /* Pid context */
unsigned long when; /* When did the operation occur */ unsigned long when; /* When did the operation occur */
...@@ -290,7 +295,7 @@ static inline struct kmem_cache_order_objects oo_make(int order, ...@@ -290,7 +295,7 @@ static inline struct kmem_cache_order_objects oo_make(int order,
unsigned long size) unsigned long size)
{ {
struct kmem_cache_order_objects x = { struct kmem_cache_order_objects x = {
(order << 16) + (PAGE_SIZE << order) / size (order << OO_SHIFT) + (PAGE_SIZE << order) / size
}; };
return x; return x;
...@@ -298,12 +303,12 @@ static inline struct kmem_cache_order_objects oo_make(int order, ...@@ -298,12 +303,12 @@ static inline struct kmem_cache_order_objects oo_make(int order,
static inline int oo_order(struct kmem_cache_order_objects x) static inline int oo_order(struct kmem_cache_order_objects x)
{ {
return x.x >> 16; return x.x >> OO_SHIFT;
} }
static inline int oo_objects(struct kmem_cache_order_objects x) static inline int oo_objects(struct kmem_cache_order_objects x)
{ {
return x.x & ((1 << 16) - 1); return x.x & OO_MASK;
} }
#ifdef CONFIG_SLUB_DEBUG #ifdef CONFIG_SLUB_DEBUG
...@@ -367,7 +372,7 @@ static struct track *get_track(struct kmem_cache *s, void *object, ...@@ -367,7 +372,7 @@ static struct track *get_track(struct kmem_cache *s, void *object,
} }
static void set_track(struct kmem_cache *s, void *object, static void set_track(struct kmem_cache *s, void *object,
enum track_item alloc, void *addr) enum track_item alloc, unsigned long addr)
{ {
struct track *p; struct track *p;
...@@ -391,8 +396,8 @@ static void init_tracking(struct kmem_cache *s, void *object) ...@@ -391,8 +396,8 @@ static void init_tracking(struct kmem_cache *s, void *object)
if (!(s->flags & SLAB_STORE_USER)) if (!(s->flags & SLAB_STORE_USER))
return; return;
set_track(s, object, TRACK_FREE, NULL); set_track(s, object, TRACK_FREE, 0UL);
set_track(s, object, TRACK_ALLOC, NULL); set_track(s, object, TRACK_ALLOC, 0UL);
} }
static void print_track(const char *s, struct track *t) static void print_track(const char *s, struct track *t)
...@@ -401,7 +406,7 @@ static void print_track(const char *s, struct track *t) ...@@ -401,7 +406,7 @@ static void print_track(const char *s, struct track *t)
return; return;
printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n", printk(KERN_ERR "INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
s, t->addr, jiffies - t->when, t->cpu, t->pid); s, (void *)t->addr, jiffies - t->when, t->cpu, t->pid);
} }
static void print_tracking(struct kmem_cache *s, void *object) static void print_tracking(struct kmem_cache *s, void *object)
...@@ -692,7 +697,7 @@ static int check_object(struct kmem_cache *s, struct page *page, ...@@ -692,7 +697,7 @@ static int check_object(struct kmem_cache *s, struct page *page,
if (!check_valid_pointer(s, page, get_freepointer(s, p))) { if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
object_err(s, page, p, "Freepointer corrupt"); object_err(s, page, p, "Freepointer corrupt");
/* /*
* No choice but to zap it and thus loose the remainder * No choice but to zap it and thus lose the remainder
* of the free objects in this slab. May cause * of the free objects in this slab. May cause
* another error because the object count is now wrong. * another error because the object count is now wrong.
*/ */
...@@ -764,8 +769,8 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search) ...@@ -764,8 +769,8 @@ static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
} }
max_objects = (PAGE_SIZE << compound_order(page)) / s->size; max_objects = (PAGE_SIZE << compound_order(page)) / s->size;
if (max_objects > 65535) if (max_objects > MAX_OBJS_PER_PAGE)
max_objects = 65535; max_objects = MAX_OBJS_PER_PAGE;
if (page->objects != max_objects) { if (page->objects != max_objects) {
slab_err(s, page, "Wrong number of objects. Found %d but " slab_err(s, page, "Wrong number of objects. Found %d but "
...@@ -866,7 +871,7 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page, ...@@ -866,7 +871,7 @@ static void setup_object_debug(struct kmem_cache *s, struct page *page,
} }
static int alloc_debug_processing(struct kmem_cache *s, struct page *page, static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
void *object, void *addr) void *object, unsigned long addr)
{ {
if (!check_slab(s, page)) if (!check_slab(s, page))
goto bad; goto bad;
...@@ -906,7 +911,7 @@ static int alloc_debug_processing(struct kmem_cache *s, struct page *page, ...@@ -906,7 +911,7 @@ static int alloc_debug_processing(struct kmem_cache *s, struct page *page,
} }
static int free_debug_processing(struct kmem_cache *s, struct page *page, static int free_debug_processing(struct kmem_cache *s, struct page *page,
void *object, void *addr) void *object, unsigned long addr)
{ {
if (!check_slab(s, page)) if (!check_slab(s, page))
goto fail; goto fail;
...@@ -1029,10 +1034,10 @@ static inline void setup_object_debug(struct kmem_cache *s, ...@@ -1029,10 +1034,10 @@ static inline void setup_object_debug(struct kmem_cache *s,
struct page *page, void *object) {} struct page *page, void *object) {}
static inline int alloc_debug_processing(struct kmem_cache *s, static inline int alloc_debug_processing(struct kmem_cache *s,
struct page *page, void *object, void *addr) { return 0; } struct page *page, void *object, unsigned long addr) { return 0; }
static inline int free_debug_processing(struct kmem_cache *s, static inline int free_debug_processing(struct kmem_cache *s,
struct page *page, void *object, void *addr) { return 0; } struct page *page, void *object, unsigned long addr) { return 0; }
static inline int slab_pad_check(struct kmem_cache *s, struct page *page) static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
{ return 1; } { return 1; }
...@@ -1499,8 +1504,8 @@ static inline int node_match(struct kmem_cache_cpu *c, int node) ...@@ -1499,8 +1504,8 @@ static inline int node_match(struct kmem_cache_cpu *c, int node)
* we need to allocate a new slab. This is the slowest path since it involves * we need to allocate a new slab. This is the slowest path since it involves
* a call to the page allocator and the setup of a new slab. * a call to the page allocator and the setup of a new slab.
*/ */
static void *__slab_alloc(struct kmem_cache *s, static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
gfp_t gfpflags, int node, void *addr, struct kmem_cache_cpu *c) unsigned long addr, struct kmem_cache_cpu *c)
{ {
void **object; void **object;
struct page *new; struct page *new;
...@@ -1584,13 +1589,18 @@ static void *__slab_alloc(struct kmem_cache *s, ...@@ -1584,13 +1589,18 @@ static void *__slab_alloc(struct kmem_cache *s,
* Otherwise we can simply pick the next object from the lockless free list. * Otherwise we can simply pick the next object from the lockless free list.
*/ */
static __always_inline void *slab_alloc(struct kmem_cache *s, static __always_inline void *slab_alloc(struct kmem_cache *s,
gfp_t gfpflags, int node, void *addr) gfp_t gfpflags, int node, unsigned long addr)
{ {
void **object; void **object;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
unsigned long flags; unsigned long flags;
unsigned int objsize; unsigned int objsize;
might_sleep_if(gfpflags & __GFP_WAIT);
if (should_failslab(s->objsize, gfpflags))
return NULL;
local_irq_save(flags); local_irq_save(flags);
c = get_cpu_slab(s, smp_processor_id()); c = get_cpu_slab(s, smp_processor_id());
objsize = c->objsize; objsize = c->objsize;
...@@ -1613,14 +1623,14 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, ...@@ -1613,14 +1623,14 @@ static __always_inline void *slab_alloc(struct kmem_cache *s,
void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
{ {
return slab_alloc(s, gfpflags, -1, __builtin_return_address(0)); return slab_alloc(s, gfpflags, -1, _RET_IP_);
} }
EXPORT_SYMBOL(kmem_cache_alloc); EXPORT_SYMBOL(kmem_cache_alloc);
#ifdef CONFIG_NUMA #ifdef CONFIG_NUMA
void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
{ {
return slab_alloc(s, gfpflags, node, __builtin_return_address(0)); return slab_alloc(s, gfpflags, node, _RET_IP_);
} }
EXPORT_SYMBOL(kmem_cache_alloc_node); EXPORT_SYMBOL(kmem_cache_alloc_node);
#endif #endif
...@@ -1634,7 +1644,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node); ...@@ -1634,7 +1644,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_node);
* handling required then we can return immediately. * handling required then we can return immediately.
*/ */
static void __slab_free(struct kmem_cache *s, struct page *page, static void __slab_free(struct kmem_cache *s, struct page *page,
void *x, void *addr, unsigned int offset) void *x, unsigned long addr, unsigned int offset)
{ {
void *prior; void *prior;
void **object = (void *)x; void **object = (void *)x;
...@@ -1704,7 +1714,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page, ...@@ -1704,7 +1714,7 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
* with all sorts of special processing. * with all sorts of special processing.
*/ */
static __always_inline void slab_free(struct kmem_cache *s, static __always_inline void slab_free(struct kmem_cache *s,
struct page *page, void *x, void *addr) struct page *page, void *x, unsigned long addr)
{ {
void **object = (void *)x; void **object = (void *)x;
struct kmem_cache_cpu *c; struct kmem_cache_cpu *c;
...@@ -1731,11 +1741,11 @@ void kmem_cache_free(struct kmem_cache *s, void *x) ...@@ -1731,11 +1741,11 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
page = virt_to_head_page(x); page = virt_to_head_page(x);
slab_free(s, page, x, __builtin_return_address(0)); slab_free(s, page, x, _RET_IP_);
} }
EXPORT_SYMBOL(kmem_cache_free); EXPORT_SYMBOL(kmem_cache_free);
/* Figure out on which slab object the object resides */ /* Figure out on which slab page the object resides */
static struct page *get_object_page(const void *x) static struct page *get_object_page(const void *x)
{ {
struct page *page = virt_to_head_page(x); struct page *page = virt_to_head_page(x);
...@@ -1807,8 +1817,8 @@ static inline int slab_order(int size, int min_objects, ...@@ -1807,8 +1817,8 @@ static inline int slab_order(int size, int min_objects,
int rem; int rem;
int min_order = slub_min_order; int min_order = slub_min_order;
if ((PAGE_SIZE << min_order) / size > 65535) if ((PAGE_SIZE << min_order) / size > MAX_OBJS_PER_PAGE)
return get_order(size * 65535) - 1; return get_order(size * MAX_OBJS_PER_PAGE) - 1;
for (order = max(min_order, for (order = max(min_order,
fls(min_objects * size - 1) - PAGE_SHIFT); fls(min_objects * size - 1) - PAGE_SHIFT);
...@@ -2073,8 +2083,7 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags) ...@@ -2073,8 +2083,7 @@ static inline int alloc_kmem_cache_cpus(struct kmem_cache *s, gfp_t flags)
* when allocating for the kmalloc_node_cache. This is used for bootstrapping * when allocating for the kmalloc_node_cache. This is used for bootstrapping
* memory on a fresh node that has no slab structures yet. * memory on a fresh node that has no slab structures yet.
*/ */
static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags, static void early_kmem_cache_node_alloc(gfp_t gfpflags, int node)
int node)
{ {
struct page *page; struct page *page;
struct kmem_cache_node *n; struct kmem_cache_node *n;
...@@ -2112,7 +2121,6 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags, ...@@ -2112,7 +2121,6 @@ static struct kmem_cache_node *early_kmem_cache_node_alloc(gfp_t gfpflags,
local_irq_save(flags); local_irq_save(flags);
add_partial(n, page, 0); add_partial(n, page, 0);
local_irq_restore(flags); local_irq_restore(flags);
return n;
} }
static void free_kmem_cache_nodes(struct kmem_cache *s) static void free_kmem_cache_nodes(struct kmem_cache *s)
...@@ -2144,8 +2152,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) ...@@ -2144,8 +2152,7 @@ static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
n = &s->local_node; n = &s->local_node;
else { else {
if (slab_state == DOWN) { if (slab_state == DOWN) {
n = early_kmem_cache_node_alloc(gfpflags, early_kmem_cache_node_alloc(gfpflags, node);
node);
continue; continue;
} }
n = kmem_cache_alloc_node(kmalloc_caches, n = kmem_cache_alloc_node(kmalloc_caches,
...@@ -2659,7 +2666,7 @@ void *__kmalloc(size_t size, gfp_t flags) ...@@ -2659,7 +2666,7 @@ void *__kmalloc(size_t size, gfp_t flags)
if (unlikely(ZERO_OR_NULL_PTR(s))) if (unlikely(ZERO_OR_NULL_PTR(s)))
return s; return s;
return slab_alloc(s, flags, -1, __builtin_return_address(0)); return slab_alloc(s, flags, -1, _RET_IP_);
} }
EXPORT_SYMBOL(__kmalloc); EXPORT_SYMBOL(__kmalloc);
...@@ -2687,7 +2694,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node) ...@@ -2687,7 +2694,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
if (unlikely(ZERO_OR_NULL_PTR(s))) if (unlikely(ZERO_OR_NULL_PTR(s)))
return s; return s;
return slab_alloc(s, flags, node, __builtin_return_address(0)); return slab_alloc(s, flags, node, _RET_IP_);
} }
EXPORT_SYMBOL(__kmalloc_node); EXPORT_SYMBOL(__kmalloc_node);
#endif #endif
...@@ -2744,7 +2751,7 @@ void kfree(const void *x) ...@@ -2744,7 +2751,7 @@ void kfree(const void *x)
put_page(page); put_page(page);
return; return;
} }
slab_free(page->slab, page, object, __builtin_return_address(0)); slab_free(page->slab, page, object, _RET_IP_);
} }
EXPORT_SYMBOL(kfree); EXPORT_SYMBOL(kfree);
...@@ -3123,8 +3130,12 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, ...@@ -3123,8 +3130,12 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *))); s->inuse = max_t(int, s->inuse, ALIGN(size, sizeof(void *)));
up_write(&slub_lock); up_write(&slub_lock);
if (sysfs_slab_alias(s, name)) if (sysfs_slab_alias(s, name)) {
down_write(&slub_lock);
s->refcount--;
up_write(&slub_lock);
goto err; goto err;
}
return s; return s;
} }
...@@ -3134,8 +3145,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size, ...@@ -3134,8 +3145,13 @@ struct kmem_cache *kmem_cache_create(const char *name, size_t size,
size, align, flags, ctor)) { size, align, flags, ctor)) {
list_add(&s->list, &slab_caches); list_add(&s->list, &slab_caches);
up_write(&slub_lock); up_write(&slub_lock);
if (sysfs_slab_add(s)) if (sysfs_slab_add(s)) {
down_write(&slub_lock);
list_del(&s->list);
up_write(&slub_lock);
kfree(s);
goto err; goto err;
}
return s; return s;
} }
kfree(s); kfree(s);
...@@ -3202,7 +3218,7 @@ static struct notifier_block __cpuinitdata slab_notifier = { ...@@ -3202,7 +3218,7 @@ static struct notifier_block __cpuinitdata slab_notifier = {
#endif #endif
void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller) void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
{ {
struct kmem_cache *s; struct kmem_cache *s;
...@@ -3218,7 +3234,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller) ...@@ -3218,7 +3234,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, void *caller)
} }
void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
int node, void *caller) int node, unsigned long caller)
{ {
struct kmem_cache *s; struct kmem_cache *s;
...@@ -3429,7 +3445,7 @@ static void resiliency_test(void) {}; ...@@ -3429,7 +3445,7 @@ static void resiliency_test(void) {};
struct location { struct location {
unsigned long count; unsigned long count;
void *addr; unsigned long addr;
long long sum_time; long long sum_time;
long min_time; long min_time;
long max_time; long max_time;
...@@ -3477,7 +3493,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s, ...@@ -3477,7 +3493,7 @@ static int add_location(struct loc_track *t, struct kmem_cache *s,
{ {
long start, end, pos; long start, end, pos;
struct location *l; struct location *l;
void *caddr; unsigned long caddr;
unsigned long age = jiffies - track->when; unsigned long age = jiffies - track->when;
start = -1; start = -1;
...@@ -4345,7 +4361,7 @@ static void sysfs_slab_remove(struct kmem_cache *s) ...@@ -4345,7 +4361,7 @@ static void sysfs_slab_remove(struct kmem_cache *s)
/* /*
* Need to buffer aliases during bootup until sysfs becomes * Need to buffer aliases during bootup until sysfs becomes
* available lest we loose that information. * available lest we lose that information.
*/ */
struct saved_alias { struct saved_alias {
struct kmem_cache *s; struct kmem_cache *s;
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment