diff options
| -rw-r--r-- | mm/workingset.c | 30 |
1 files changed, 29 insertions, 1 deletions
diff --git a/mm/workingset.c b/mm/workingset.c index f874b2c663e3..9a26a60368d2 100644 --- a/mm/workingset.c +++ b/mm/workingset.c @@ -156,8 +156,19 @@ ZONES_SHIFT + NODES_SHIFT) #define EVICTION_MASK (~0UL >> EVICTION_SHIFT) +/* + * Eviction timestamps need to be able to cover the full range of + * actionable refaults. However, bits are tight in the radix tree + * entry, and after storing the identifier for the lruvec there might + * not be enough left to represent every single actionable refault. In + * that case, we have to sacrifice granularity for distance, and group + * evictions into coarser buckets by shaving off lower timestamp bits. + */ +static unsigned int bucket_order __read_mostly; + static void *pack_shadow(unsigned long eviction, struct zone *zone) { + eviction >>= bucket_order; eviction = (eviction << NODES_SHIFT) | zone_to_nid(zone); eviction = (eviction << ZONES_SHIFT) | zone_idx(zone); eviction = (eviction << RADIX_TREE_EXCEPTIONAL_SHIFT); @@ -178,7 +189,7 @@ static void unpack_shadow(void *shadow, struct zone **zonep, entry >>= NODES_SHIFT; *zonep = NODE_DATA(nid)->node_zones + zid; - *evictionp = entry; + *evictionp = entry << bucket_order; } /** @@ -400,8 +411,25 @@ static struct lock_class_key shadow_nodes_key; static int __init workingset_init(void) { + unsigned int timestamp_bits; + unsigned int max_order; int ret; + BUILD_BUG_ON(BITS_PER_LONG < EVICTION_SHIFT); + /* + * Calculate the eviction bucket size to cover the longest + * actionable refault distance, which is currently half of + * memory (totalram_pages/2). However, memory hotplug may add + * some more pages at runtime, so keep working with up to + * double the initial memory by using totalram_pages as-is. + */ + timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT; + max_order = fls_long(totalram_pages - 1); + if (max_order > timestamp_bits) + bucket_order = max_order - timestamp_bits; + printk("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n", + timestamp_bits, max_order, bucket_order); + ret = list_lru_init_key(&workingset_shadow_nodes, &shadow_nodes_key); if (ret) goto err; |