[PATCH v2 08/10] mm/madvise: separate the PTL-held PTE scan

[Gregory Price (Meta)]

The PTE loop's scan is broken up by the two places that drop the
PTE lock: the periodic reschedule and the large-folio split.

Move the scan into madvise_lru_pte_range_locked(), which runs
entirely under the lock and stops either at a folio to split or
at the reschedule point.

Both lock drops now happen in madvise_lru_pmd_entry(), which restarts
the scan where the helper stopped, or past the folio's PTE batch if
splitting it failed.

No functional change intended.

Signed-off-by: Gregory Price (Meta) <gourry@xxxxxxxxxx>
---
 mm/madvise.c | 67 ++++++++++++++++++++++++++++++++--------------------
 1 file changed, 42 insertions(+), 25 deletions(-)

diff --git a/mm/madvise.c b/mm/madvise.c
index 0c0a840102eb..3ebc14d30e0b 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -565,6 +565,33 @@ static bool madvise_lru_huge_pmd(pmd_t *pmd, unsigned long addr,
 }
 #endif
 
+/*
+ * Scan PTEs under the PTE lock until a folio needs splitting, and return it
+ * locked and referenced. Return NULL at the end of the range, or earlier
+ * when the caller should reschedule. *addr is left at the PTE where the
+ * scan stopped.
+ */
+static struct folio *madvise_lru_pte_range_locked(pte_t *pte,
+		unsigned long *addr, unsigned long end, struct mm_walk *walk,
+		struct list_head *folio_list, bool pageout_anon_only, int *nr,
+		unsigned int *batch_count)
+{
+	struct folio *folio;
+
+	for (; *addr < end; pte += *nr, *addr += *nr * PAGE_SIZE) {
+		if (++(*batch_count) == SWAP_CLUSTER_MAX) {
+			*batch_count = 0;
+			if (need_resched())
+				return NULL;
+		}
+		folio = madvise_lru_pte_batch_locked(pte, *addr, end, walk,
+				folio_list, pageout_anon_only, nr);
+		if (folio)
+			return folio;
+	}
+	return NULL;
+}
+
 static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr,
 		unsigned long end, struct mm_walk *walk)
 {
@@ -596,34 +623,24 @@ static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr,
 		goto out;
 	flush_tlb_batched_pending(mm);
 	lazy_mmu_mode_enable();
-	for (; addr < end; pte += nr, addr += nr * PAGE_SIZE) {
-		if (++batch_count == SWAP_CLUSTER_MAX) {
-			batch_count = 0;
-			if (need_resched()) {
-				lazy_mmu_mode_disable();
-				pte_unmap_unlock(start_pte, ptl);
-				cond_resched();
-				goto restart;
-			}
-		}
-
-		folio = madvise_lru_pte_batch_locked(pte, addr, end, walk,
-				&folio_list, pageout_anon_only, &nr);
-		if (!folio)
-			continue;
-
+	folio = madvise_lru_pte_range_locked(pte, &addr, end, walk,
+			&folio_list, pageout_anon_only, &nr, &batch_count);
+	if (!folio && addr < end) {
 		lazy_mmu_mode_disable();
 		pte_unmap_unlock(start_pte, ptl);
-		start_pte = NULL;
-		if (!madvise_lru_split_folio(folio))
-			nr = 0;
-		start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
-		if (!start_pte)
-			break;
-		pte = start_pte;
-		flush_tlb_batched_pending(mm);
-		lazy_mmu_mode_enable();
+		cond_resched();
+		goto restart;
 	}
+	if (!folio)
+		goto out;
+
+	lazy_mmu_mode_disable();
+	pte_unmap_unlock(start_pte, ptl);
+	start_pte = NULL;
+	if (!madvise_lru_split_folio(folio))
+		nr = 0;
+	addr += nr * PAGE_SIZE;
+	goto restart;
 
 out:
 	if (start_pte) {
-- 
2.56.0