[PATCH] mm/slub: batch-detach node partial slabs

[Hao Li]

get_partial_node_bulk() used to move each selected slab from the node
partial list to the local pc->slabs list using a remove_partial() and
list_add() pair. In practice, the loop often detaches several adjacent
slabs, so this repeatedly manipulates list pointers while holding
n->list_lock, which causes unnecessary churn.

Instead, track contiguous runs of matching slabs and move each run with
list_bulk_move_tail() in one operation. This reduces list pointer churn
inside the lock critical section.

The mmap2 testcase shows a 5% improvement after applying this patch.

Signed-off-by: Hao Li <hao.li@xxxxxxxxx>
---
 mm/slub.c | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 04692a6f9128..180973a4a3d2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3739,6 +3739,7 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 				  bool allow_spin)
 {
 	struct slab *slab, *slab2;
+	struct slab *first = NULL, *last = NULL;
 	unsigned int total_free = 0;
 	unsigned long flags;
 
@@ -3757,8 +3758,15 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 		struct freelist_counters flc;
 		unsigned int slab_free;
 
-		if (!pfmemalloc_match(slab, pc->flags))
+		if (!pfmemalloc_match(slab, pc->flags)) {
+			if (first) {
+				list_bulk_move_tail(&pc->slabs,
+						    &first->slab_list,
+						    &last->slab_list);
+				first = NULL;
+			}
 			continue;
+		}
 
 		/*
 		 * determine the number of free objects in the slab racily
@@ -3775,15 +3783,21 @@ static bool get_partial_node_bulk(struct kmem_cache *s,
 		    && total_free + slab_free > pc->max_objects)
 			break;
 
-		remove_partial(n, slab);
-
-		list_add(&slab->slab_list, &pc->slabs);
+		if (!first)
+			first = slab;
+		last = slab;
+		slab_clear_node_partial(slab);
+		n->nr_partial--;
 
 		total_free += slab_free;
 		if (total_free >= pc->max_objects)
 			break;
 	}
 
+	if (first)
+		list_bulk_move_tail(&pc->slabs, &first->slab_list,
+				    &last->slab_list);
+
 	spin_unlock_irqrestore(&n->list_lock, flags);
 	return total_free > 0;
 }
-- 
2.54.0