Commit b5ff4133 authored by T.J. Alumbaugh's avatar T.J. Alumbaugh Committed by Andrew Morton
Browse files

mm: multi-gen LRU: improve walk_pmd_range() 

Improve readability of walk_pmd_range() and walk_pmd_range_locked().

Link: https://lkml.kernel.org/r/20230118001827.1040870-7-talumbau@google.com


Signed-off-by: default avatarT.J. Alumbaugh <talumbau@google.com>
Cc: Yu Zhao <yuzhao@google.com>
Signed-off-by: default avatarAndrew Morton <akpm@linux-foundation.org>
parent 37cc9997

mm/vmscan.c

+20 −20
Original line number Diff line number Diff line
@@ -3999,8 +3999,8 @@ static bool walk_pte_range(pmd_t *pmd, unsigned long start, unsigned long end, 
}



#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG)

static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,

				  struct mm_walk *args, unsigned long *bitmap, unsigned long *start)

static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,

				  struct mm_walk *args, unsigned long *bitmap, unsigned long *first)

{

	int i;

	pmd_t *pmd;
@@ -4013,18 +4013,19 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area 
	VM_WARN_ON_ONCE(pud_leaf(*pud));



	/* try to batch at most 1+MIN_LRU_BATCH+1 entries */

	if (*start == -1) {

		*start = next;

	if (*first == -1) {

		*first = addr;

		bitmap_zero(bitmap, MIN_LRU_BATCH);

		return;

	}



	i = next == -1 ? 0 : pmd_index(next) - pmd_index(*start);

	i = addr == -1 ? 0 : pmd_index(addr) - pmd_index(*first);

	if (i && i <= MIN_LRU_BATCH) {

		__set_bit(i - 1, bitmap);

		return;

	}



	pmd = pmd_offset(pud, *start);

	pmd = pmd_offset(pud, *first);



	ptl = pmd_lockptr(args->mm, pmd);

	if (!spin_trylock(ptl))
@@ -4035,15 +4036,16 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area 
	do {

		unsigned long pfn;

		struct folio *folio;

		unsigned long addr = i ? (*start & PMD_MASK) + i * PMD_SIZE : *start;



		/* don't round down the first address */

		addr = i ? (*first & PMD_MASK) + i * PMD_SIZE : *first;



		pfn = get_pmd_pfn(pmd[i], vma, addr);

		if (pfn == -1)

			goto next;



		if (!pmd_trans_huge(pmd[i])) {

			if (arch_has_hw_nonleaf_pmd_young() &&

			    get_cap(LRU_GEN_NONLEAF_YOUNG))

			if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG))

				pmdp_test_and_clear_young(vma, addr, pmd + i);

			goto next;

		}
@@ -4072,12 +4074,11 @@ static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area 
	arch_leave_lazy_mmu_mode();

	spin_unlock(ptl);

done:

	*start = -1;

	bitmap_zero(bitmap, MIN_LRU_BATCH);

	*first = -1;

}

#else

static void walk_pmd_range_locked(pud_t *pud, unsigned long next, struct vm_area_struct *vma,

				  struct mm_walk *args, unsigned long *bitmap, unsigned long *start)

static void walk_pmd_range_locked(pud_t *pud, unsigned long addr, struct vm_area_struct *vma,

				  struct mm_walk *args, unsigned long *bitmap, unsigned long *first)

{

}

#endif
@@ -4090,9 +4091,9 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 
	unsigned long next;

	unsigned long addr;

	struct vm_area_struct *vma;

	unsigned long pos = -1;

	unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)];

	unsigned long first = -1;

	struct lru_gen_mm_walk *walk = args->private;

	unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {};



	VM_WARN_ON_ONCE(pud_leaf(*pud));
@@ -4131,18 +4132,17 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 
			if (pfn < pgdat->node_start_pfn || pfn >= pgdat_end_pfn(pgdat))

				continue;



			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);

			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);

			continue;

		}

#endif

		walk->mm_stats[MM_NONLEAF_TOTAL]++;



		if (arch_has_hw_nonleaf_pmd_young() &&

		    get_cap(LRU_GEN_NONLEAF_YOUNG)) {

		if (arch_has_hw_nonleaf_pmd_young() && get_cap(LRU_GEN_NONLEAF_YOUNG)) {

			if (!pmd_young(val))

				continue;



			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &pos);

			walk_pmd_range_locked(pud, addr, vma, args, bitmap, &first);

		}



		if (!walk->force_scan && !test_bloom_filter(walk->lruvec, walk->max_seq, pmd + i))
@@ -4159,7 +4159,7 @@ static void walk_pmd_range(pud_t *pud, unsigned long start, unsigned long end, 
		update_bloom_filter(walk->lruvec, walk->max_seq + 1, pmd + i);

	}



	walk_pmd_range_locked(pud, -1, vma, args, bitmap, &pos);

	walk_pmd_range_locked(pud, -1, vma, args, bitmap, &first);



	if (i < PTRS_PER_PMD && get_next_vma(PUD_MASK, PMD_SIZE, args, &start, &end))

		goto restart;