Commit 889a3747 authored by Matthew Wilcox (Oracle)'s avatar Matthew Wilcox (Oracle)
Browse files

mm/lru: Add folio LRU functions 



Handle arbitrary-order folios being added to the LRU.  By definition,
all pages being added to the LRU were already head or base pages, but
call page_folio() on them anyway to get the type right and avoid the
buried calls to compound_head().

Saves 783 bytes of kernel text; no functions grow.

Signed-off-by: default avatarMatthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: default avatarYu Zhao <yuzhao@google.com>
Reviewed-by: default avatarChristoph Hellwig <hch@lst.de>
Reviewed-by: default avatarDavid Howells <dhowells@redhat.com>
Acked-by: default avatarKirill A. Shutemov <kirill.shutemov@linux.intel.com>
Acked-by: default avatarMike Rapoport <rppt@linux.ibm.com>
Acked-by: default avatarVlastimil Babka <vbabka@suse.cz>
parent d389a4a8

Documentation/core-api/mm-api.rst

+1 −0
Original line number Diff line number Diff line
@@ -95,6 +95,7 @@ More Memory Management Functions 
.. kernel-doc:: mm/mempolicy.c

.. kernel-doc:: include/linux/mm_types.h

   :internal:

.. kernel-doc:: include/linux/mm_inline.h

.. kernel-doc:: include/linux/page-flags.h

.. kernel-doc:: include/linux/mm.h

   :internal:

include/linux/mm_inline.h

+67 −36
Original line number Diff line number Diff line
@@ -6,27 +6,33 @@ 
#include <linux/swap.h>



/**

 * page_is_file_lru - should the page be on a file LRU or anon LRU?

 * @page: the page to test

 *

 * Returns 1 if @page is a regular filesystem backed page cache page or a lazily

 * freed anonymous page (e.g. via MADV_FREE).  Returns 0 if @page is a normal

 * anonymous page, a tmpfs page or otherwise ram or swap backed page.  Used by

 * functions that manipulate the LRU lists, to sort a page onto the right LRU

 * list.

 * folio_is_file_lru - Should the folio be on a file LRU or anon LRU?

 * @folio: The folio to test.

 *

 * We would like to get this info without a page flag, but the state

 * needs to survive until the page is last deleted from the LRU, which

 * needs to survive until the folio is last deleted from the LRU, which

 * could be as far down as __page_cache_release.

 *

 * Return: An integer (not a boolean!) used to sort a folio onto the

 * right LRU list and to account folios correctly.

 * 1 if @folio is a regular filesystem backed page cache folio

 * or a lazily freed anonymous folio (e.g. via MADV_FREE).

 * 0 if @folio is a normal anonymous folio, a tmpfs folio or otherwise

 * ram or swap backed folio.

 */

static inline int folio_is_file_lru(struct folio *folio)

{

	return !folio_test_swapbacked(folio);

}



static inline int page_is_file_lru(struct page *page)

{

	return !PageSwapBacked(page);

	return folio_is_file_lru(page_folio(page));

}



static __always_inline void update_lru_size(struct lruvec *lruvec,

				enum lru_list lru, enum zone_type zid,

				int nr_pages)

				long nr_pages)

{

	struct pglist_data *pgdat = lruvec_pgdat(lruvec);
@@ -39,69 +45,94 @@ static __always_inline void update_lru_size(struct lruvec *lruvec, 
}



/**

 * __clear_page_lru_flags - clear page lru flags before releasing a page

 * @page: the page that was on lru and now has a zero reference

 * __folio_clear_lru_flags - Clear page lru flags before releasing a page.

 * @folio: The folio that was on lru and now has a zero reference.

 */

static __always_inline void __clear_page_lru_flags(struct page *page)

static __always_inline void __folio_clear_lru_flags(struct folio *folio)

{

	VM_BUG_ON_PAGE(!PageLRU(page), page);

	VM_BUG_ON_FOLIO(!folio_test_lru(folio), folio);



	__ClearPageLRU(page);

	__folio_clear_lru(folio);



	/* this shouldn't happen, so leave the flags to bad_page() */

	if (PageActive(page) && PageUnevictable(page))

	if (folio_test_active(folio) && folio_test_unevictable(folio))

		return;



	__ClearPageActive(page);

	__ClearPageUnevictable(page);

	__folio_clear_active(folio);

	__folio_clear_unevictable(folio);

}



static __always_inline void __clear_page_lru_flags(struct page *page)

{

	__folio_clear_lru_flags(page_folio(page));

}



/**

 * page_lru - which LRU list should a page be on?

 * @page: the page to test

 * folio_lru_list - Which LRU list should a folio be on?

 * @folio: The folio to test.

 *

 * Returns the LRU list a page should be on, as an index

 * Return: The LRU list a folio should be on, as an index

 * into the array of LRU lists.

 */

static __always_inline enum lru_list page_lru(struct page *page)

static __always_inline enum lru_list folio_lru_list(struct folio *folio)

{

	enum lru_list lru;



	VM_BUG_ON_PAGE(PageActive(page) && PageUnevictable(page), page);

	VM_BUG_ON_FOLIO(folio_test_active(folio) && folio_test_unevictable(folio), folio);



	if (PageUnevictable(page))

	if (folio_test_unevictable(folio))

		return LRU_UNEVICTABLE;



	lru = page_is_file_lru(page) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;

	if (PageActive(page))

	lru = folio_is_file_lru(folio) ? LRU_INACTIVE_FILE : LRU_INACTIVE_ANON;

	if (folio_test_active(folio))

		lru += LRU_ACTIVE;



	return lru;

}



static __always_inline

void lruvec_add_folio(struct lruvec *lruvec, struct folio *folio)

{

	enum lru_list lru = folio_lru_list(folio);



	update_lru_size(lruvec, lru, folio_zonenum(folio),

			folio_nr_pages(folio));

	list_add(&folio->lru, &lruvec->lists[lru]);

}



static __always_inline void add_page_to_lru_list(struct page *page,

				struct lruvec *lruvec)

{

	enum lru_list lru = page_lru(page);

	lruvec_add_folio(lruvec, page_folio(page));

}



	update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page));

	list_add(&page->lru, &lruvec->lists[lru]);

static __always_inline

void lruvec_add_folio_tail(struct lruvec *lruvec, struct folio *folio)

{

	enum lru_list lru = folio_lru_list(folio);



	update_lru_size(lruvec, lru, folio_zonenum(folio),

			folio_nr_pages(folio));

	list_add_tail(&folio->lru, &lruvec->lists[lru]);

}



static __always_inline void add_page_to_lru_list_tail(struct page *page,

				struct lruvec *lruvec)

{

	enum lru_list lru = page_lru(page);

	lruvec_add_folio_tail(lruvec, page_folio(page));

}



	update_lru_size(lruvec, lru, page_zonenum(page), thp_nr_pages(page));

	list_add_tail(&page->lru, &lruvec->lists[lru]);

static __always_inline

void lruvec_del_folio(struct lruvec *lruvec, struct folio *folio)

{

	list_del(&folio->lru);

	update_lru_size(lruvec, folio_lru_list(folio), folio_zonenum(folio),

			-folio_nr_pages(folio));

}



static __always_inline void del_page_from_lru_list(struct page *page,

				struct lruvec *lruvec)

{

	list_del(&page->lru);

	update_lru_size(lruvec, page_lru(page), page_zonenum(page),

			-thp_nr_pages(page));

	lruvec_del_folio(lruvec, page_folio(page));

}

#endif

include/trace/events/pagemap.h

+1 −1
Original line number Diff line number Diff line
@@ -41,7 +41,7 @@ TRACE_EVENT(mm_lru_insertion, 
	TP_fast_assign(

		__entry->page	= page;

		__entry->pfn	= page_to_pfn(page);

		__entry->lru	= page_lru(page);

		__entry->lru	= folio_lru_list(page_folio(page));

		__entry->flags	= trace_pagemap_flags(page);

	),