From 5cf909c553e9efed573811de4b3f5172898d5515 Mon Sep 17 00:00:00 2001 From: Oliver Glitta Date: Wed, 7 Jul 2021 18:07:47 -0700 Subject: mm/slub: use stackdepot to save stack trace in objects Many stack traces are similar so there are many similar arrays. Stackdepot saves each unique stack only once. Replace field addrs in struct track with depot_stack_handle_t handle. Use stackdepot to save stack trace. The benefits are smaller memory overhead and possibility to aggregate per-cache statistics in the following patch using the stackdepot handle instead of matching stacks manually. [ vbabka@suse.cz: rebase to 5.17-rc1 and adjust accordingly ] This was initially merged as commit 788691464c29 and reverted by commit ae14c63a9f20 due to several issues, that should now be fixed. The problem of unconditional memory overhead by stackdepot has been addressed by commit 2dba5eb1c73b ("lib/stackdepot: allow optional init and stack_table allocation by kvmalloc()"), so the dependency on stackdepot will result in extra memory usage only when a slab cache tracking is actually enabled, and not for all CONFIG_SLUB_DEBUG builds. The build failures on some architectures were also addressed, and the reported issue with xfs/433 test did not reproduce on 5.17-rc1 with this patch. Signed-off-by: Oliver Glitta Signed-off-by: Vlastimil Babka Reviewed-and-tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Acked-by: David Rientjes Cc: David Rientjes Cc: Christoph Lameter Cc: Pekka Enberg Cc: Joonsoo Kim --- mm/slab_common.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'mm/slab_common.c') diff --git a/mm/slab_common.c b/mm/slab_common.c index 6ee64d6208b3..73943479a2b7 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -24,6 +24,7 @@ #include #include #include +#include #define CREATE_TRACE_POINTS #include @@ -314,9 +315,13 @@ kmem_cache_create_usercopy(const char *name, * If no slub_debug was enabled globally, the static key is not yet * enabled by setup_slub_debug(). Enable it if the cache is being * created with any of the debugging flags passed explicitly. + * It's also possible that this is the first cache created with + * SLAB_STORE_USER and we should init stack_depot for it. */ if (flags & SLAB_DEBUG_FLAGS) static_branch_enable(&slub_debug_enabled); + if (flags & SLAB_STORE_USER) + stack_depot_init(); #endif mutex_lock(&slab_mutex); -- cgit 1.4.1 From 33647783deb674b60bb492718cef102c3e9dce6b Mon Sep 17 00:00:00 2001 From: Ohhoon Kwon Date: Mon, 11 Apr 2022 01:25:11 +0900 Subject: mm/slab_common: move dma-kmalloc caches creation into new_kmalloc_cache() There are four types of kmalloc_caches: KMALLOC_NORMAL, KMALLOC_CGROUP, KMALLOC_RECLAIM, and KMALLOC_DMA. While the first three types are created using new_kmalloc_cache(), KMALLOC_DMA caches are created in a separate logic. Let KMALLOC_DMA caches be also created using new_kmalloc_cache(), to enhance readability. Historically, there were only KMALLOC_NORMAL caches and KMALLOC_DMA caches in the first place, and they were initialized in two separate logics. However, when KMALLOC_RECLAIM was introduced in v4.20 via commit 1291523f2c1d ("mm, slab/slub: introduce kmalloc-reclaimable caches") and KMALLOC_CGROUP was introduced in v5.14 via commit 494c1dfe855e ("mm: memcg/slab: create a new set of kmalloc-cg- caches"), their creations were merged with KMALLOC_NORMAL's only. KMALLOC_DMA creation logic should be merged with them, too. By merging KMALLOC_DMA initialization with other types, the following two changes might occur: 1. The order dma-kmalloc- caches added in slab_cache list may be sorted by size. i.e. the order they appear in /proc/slabinfo may change as well. 2. slab_state will be set to UP after KMALLOC_DMA is created. In case of slub, freelist randomization is dependent on slab_state>=UP, and therefore KMALLOC_DMA cache's freelist will not be randomized in creation, but will be deferred to init_freelist_randomization(). Co-developed-by: JaeSang Yoo Signed-off-by: JaeSang Yoo Signed-off-by: Ohhoon Kwon Reviewed-by: Hyeonggon Yoo <42.hyeyoo@gmail.com> Acked-by: David Rientjes Signed-off-by: Vlastimil Babka Link: https://lore.kernel.org/r/20220410162511.656541-1-ohkwon1043@gmail.com --- mm/slab_common.c | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) (limited to 'mm/slab_common.c') diff --git a/mm/slab_common.c b/mm/slab_common.c index 6ee64d6208b3..a959d247c27b 100644 --- a/mm/slab_common.c +++ b/mm/slab_common.c @@ -849,6 +849,8 @@ new_kmalloc_cache(int idx, enum kmalloc_cache_type type, slab_flags_t flags) return; } flags |= SLAB_ACCOUNT; + } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) { + flags |= SLAB_CACHE_DMA; } kmalloc_caches[type][idx] = create_kmalloc_cache( @@ -877,7 +879,7 @@ void __init create_kmalloc_caches(slab_flags_t flags) /* * Including KMALLOC_CGROUP if CONFIG_MEMCG_KMEM defined */ - for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) { + for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) { for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { if (!kmalloc_caches[type][i]) new_kmalloc_cache(i, type, flags); @@ -898,20 +900,6 @@ void __init create_kmalloc_caches(slab_flags_t flags) /* Kmalloc array is now usable */ slab_state = UP; - -#ifdef CONFIG_ZONE_DMA - for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) { - struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i]; - - if (s) { - kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache( - kmalloc_info[i].name[KMALLOC_DMA], - kmalloc_info[i].size, - SLAB_CACHE_DMA | flags, 0, - kmalloc_info[i].size); - } - } -#endif } #endif /* !CONFIG_SLOB */ -- cgit 1.4.1