[PATCH v2 09/10] mm/madvise: make cold and pageout PTE lock ownership explicit

[Gregory Price (Meta)]

madvise_lru_pmd_entry() maps and unmaps the PTE table through a
restart label and an out label, so finding where the PTE lock is
held means following every goto.

Move the PTE path into madvise_lru_pte_range(), whose loop takes
the lock at the top of each pass and drops it before splitting or
rescheduling.

madvise_lru_pmd_entry() now only checks for a fatal signal, computes
the anon-only filter and dispatches to the huge-PMD or PTE path.

No functional change intended.

Signed-off-by: Gregory Price (Meta) <gourry@xxxxxxxxxx>
---
 mm/madvise.c | 91 +++++++++++++++++++++++++---------------------------
 1 file changed, 44 insertions(+), 47 deletions(-)

diff --git a/mm/madvise.c b/mm/madvise.c
index 3ebc14d30e0b..acfe5b945eea 100644
--- a/mm/madvise.c
+++ b/mm/madvise.c
@@ -592,64 +592,61 @@ static struct folio *madvise_lru_pte_range_locked(pte_t *pte,
 	return NULL;
 }
 
-static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr,
-		unsigned long end, struct mm_walk *walk)
+static void madvise_lru_pte_range(pmd_t *pmd, unsigned long addr,
+		unsigned long end, struct mm_walk *walk,
+		bool pageout_anon_only)
 {
-	struct madvise_walk_private *private = walk->private;
-	struct mmu_gather *tlb = private->tlb;
-	bool pageout = private->pageout;
-	struct mm_struct *mm = tlb->mm;
-	struct vm_area_struct *vma = walk->vma;
-	pte_t *start_pte, *pte;
-	spinlock_t *ptl;
-	struct folio *folio = NULL;
+	const struct madvise_walk_private *private = walk->private;
+	struct mm_struct *mm = private->tlb->mm;
 	LIST_HEAD(folio_list);
 	unsigned int batch_count = 0;
-	bool pageout_anon_only;
+	struct folio *folio;
+	pte_t *start_pte;
+	spinlock_t *ptl;
 	int nr;
 
-	if (fatal_signal_pending(current))
-		return -EINTR;
-	pageout_anon_only = pageout && !vma_is_anonymous(vma) &&
-			    !can_do_file_pageout(vma);
-
-	if (pmd_trans_huge(*pmd) &&
-	    madvise_lru_huge_pmd(pmd, addr, end, walk, pageout_anon_only))
-		return 0;
-	tlb_change_page_size(tlb, PAGE_SIZE);
-restart:
-	start_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
-	if (!start_pte)
-		goto out;
-	flush_tlb_batched_pending(mm);
-	lazy_mmu_mode_enable();
-	folio = madvise_lru_pte_range_locked(pte, &addr, end, walk,
-			&folio_list, pageout_anon_only, &nr, &batch_count);
-	if (!folio && addr < end) {
+	tlb_change_page_size(private->tlb, PAGE_SIZE);
+	while (addr < end) {
+		start_pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+		if (!start_pte)
+			break;
+		flush_tlb_batched_pending(mm);
+		lazy_mmu_mode_enable();
+		folio = madvise_lru_pte_range_locked(start_pte, &addr, end,
+				walk, &folio_list, pageout_anon_only, &nr,
+				&batch_count);
 		lazy_mmu_mode_disable();
 		pte_unmap_unlock(start_pte, ptl);
-		cond_resched();
-		goto restart;
-	}
-	if (!folio)
-		goto out;
 
-	lazy_mmu_mode_disable();
-	pte_unmap_unlock(start_pte, ptl);
-	start_pte = NULL;
-	if (!madvise_lru_split_folio(folio))
-		nr = 0;
-	addr += nr * PAGE_SIZE;
-	goto restart;
-
-out:
-	if (start_pte) {
-		lazy_mmu_mode_disable();
-		pte_unmap_unlock(start_pte, ptl);
+		if (folio) {
+			if (!madvise_lru_split_folio(folio))
+				nr = 0;
+			addr += nr * PAGE_SIZE;
+		} else if (addr < end) {
+			cond_resched();
+		}
 	}
-	if (pageout)
+	if (private->pageout)
 		reclaim_pages(&folio_list);
 	cond_resched();
+}
+
+static int madvise_lru_pmd_entry(pmd_t *pmd, unsigned long addr,
+		unsigned long next, struct mm_walk *walk)
+{
+	const struct madvise_walk_private *private = walk->private;
+	struct vm_area_struct *vma = walk->vma;
+	bool pageout_anon_only;
+
+	if (fatal_signal_pending(current))
+		return -EINTR;
+	pageout_anon_only = private->pageout && !vma_is_anonymous(vma) &&
+			    !can_do_file_pageout(vma);
+
+	if (pmd_trans_huge(*pmd) &&
+	    madvise_lru_huge_pmd(pmd, addr, next, walk, pageout_anon_only))
+		return 0;
+	madvise_lru_pte_range(pmd, addr, next, walk, pageout_anon_only);
 
 	return 0;
 }
-- 
2.56.0